From 9e3d5583ee5e270480fb659782f678add17d4899 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Mar 2018 13:32:37 -0700 Subject: [PATCH 01/98] Moved objects arround and added SSPRungeKutta time stepper Moved TimeStepper to the base component file Moved composites out of the base component file to their own file Moved concrete TimeStepper classes to the _components directory in a new file Added a strong stability preserving Runge Kutta time stepper (with no tests yet) --- .../timesteppers.py} | 135 +++++----- sympl/_core/base_components.py | 235 +++++------------- sympl/_core/composite.py | 206 +++++++++++++++ 3 files changed, 331 insertions(+), 245 deletions(-) rename sympl/{_core/timestepping.py => _components/timesteppers.py} (82%) create mode 100644 sympl/_core/composite.py diff --git a/sympl/_core/timestepping.py b/sympl/_components/timesteppers.py similarity index 82% rename from sympl/_core/timestepping.py rename to sympl/_components/timesteppers.py index ca64403..d529031 100644 --- a/sympl/_core/timestepping.py +++ b/sympl/_components/timesteppers.py @@ -1,65 +1,43 @@ -from .base_components import PrognosticComposite -import abc -from .array import DataArray - - -class TimeStepper(object): - """An object which integrates model state forward in time. - - It uses Prognostic and Diagnostic objects to update the current model state - with diagnostics, and to return the model state at the next timestep. - - Attributes - ---------- - inputs : tuple of str - The quantities required in the state when the object is called. - diagnostics: tuple of str - The quantities for which values for the old state are returned - when the object is called. - outputs: tuple of str - The quantities for which values for the new state are returned - when the object is called. +from sympl._core.base_components import TimeStepper +from .._core.array import DataArray + + +class SSPRungeKutta(TimeStepper): + """ + A TimeStepper using the Strong Stability Preserving Runge-Kutta scheme, + as in Numerical Methods for Fluid Dynamics by Dale Durran (2nd ed) and + as proposed by Shu and Osher (1988). """ - __metaclass__ = abc.ABCMeta - - def __str__(self): - return ( - 'instance of {}(TimeStepper)\n' - ' inputs: {}\n' - ' outputs: {}\n' - ' diagnostics: {}\n' - ' Prognostic components: {}'.format( - self.__class__, self.inputs, self.outputs, self.diagnostics, - str(self._prognostic)) - ) + def __init__(self, prognostic_list, stages=3): + """ + Initialize a strong stability preserving Runge-Kutta time stepper. + + Args + ---- + prognostic_list : iterable of Prognostic + Objects used to get tendencies for time stepping. + stages: int + Number of stages to use. Should be 2 or 3. + """ + if stages not in (2, 3): + raise ValueError( + 'stages must be one of 2 or 3, received {}'.format(stages)) + self._stages = stages + self._euler_stepper = AdamsBashforth(prognostic_list, order=1) + super(SSPRungeKutta, self).__init__(prognostic_list) + - def __repr__(self): - if hasattr(self, '_making_repr') and self._making_repr: - return '{}(recursive reference)'.format(self.__class__) - else: - self._making_repr = True - return_value = '{}({})'.format( - self.__class__, - '\n'.join('{}: {}'.format(repr(key), repr(value)) - for key, value in self.__dict__.items() - if key != '_making_repr')) - self._making_repr = False - return return_value - - def __init__(self, prognostic_list, **kwargs): - self._prognostic = PrognosticComposite(*prognostic_list) - - @abc.abstractmethod def __call__(self, state, timestep): """ - Retrieves any diagnostics and returns a new state corresponding + Updates the input state dictionary and returns a new state corresponding to the next timestep. Args ---- state : dict - The current model state. + The current model state. Will be updated in-place by + the call with any diagnostics from the current timestep. timestep : timedelta The amount of time to step forward. @@ -70,23 +48,42 @@ def __call__(self, state, timestep): new_state : dict The model state at the next timestep. """ - - def _copy_untouched_quantities(self, old_state, new_state): - for key in old_state.keys(): - if key not in new_state: - new_state[key] = old_state[key] - - @property - def inputs(self): - return self._prognostic.inputs - - @property - def outputs(self): - return self._prognostic.tendencies - - @property - def diagnostics(self): - return self._prognostic.diagnostics + if self._stages == 2: + return self._step_2_stages(state, timestep) + elif self._stages == 3: + return self._step_3_stages(state, timestep) + + def _step_3_stages(self, state, timestep): + diagnostics, state_1 = self._euler_stepper(state, timestep) + _, state_1_5 = self._euler_stepper(state_1, timestep) + state_2 = add(multiply(0.75, state), multiply(0.25, state_1_5)) + _, state_2_5 = self._euler_stepper(state_2, timestep) + out_state = add(multiply(1./3, state), multiply(2./3, state_2_5)) + return out_state + + + def _step_2_stages(self, state, timestep): + diagnostics, state_1 = self._euler_stepper(state, timestep) + _, state_2 = self._euler_stepper(state_1, timestep) + out_state = multiply(0.5, add(state, state_2)) + return out_state + + +def add(state_1, state_2): + out_state = {'time': state_1['time']} + for key in state.keys(): + if key != 'time': + out_state[key] = state[key] + state_2[key] + out_state[key].attrs = state_1[key].attrs + + +def multiply(scalar, state): + out_state = {'time': state['time']} + for key in state.keys(): + if key != 'time': + out_state[key] = scalar * state[key] + out_state[key].attrs = state[key].attrs + return out_state class AdamsBashforth(TimeStepper): diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 58ed688..7de58cc 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -1,6 +1,6 @@ import abc -from .util import ensure_no_shared_keys, update_dict_by_adding_another -from .exceptions import SharedKeyError + +from sympl._core.composite import PrognosticComposite class Implicit(object): @@ -410,204 +410,87 @@ def store(self, state): """ -class ComponentComposite(object): - - component_class = None +class TimeStepper(object): + """An object which integrates model state forward in time. - def __str__(self): - return '{}(\n{}\n)'.format( - self.__class__, - ',\n'.join(str(component) for component in self._components)) - - def __repr__(self): - return '{}(\n{}\n)'.format( - self.__class__, - ',\n'.join(repr(component) for component in self._components)) - - def __init__(self, *args): - """ - Args - ---- - *args - The components that should be wrapped by this object. + It uses Prognostic and Diagnostic objects to update the current model state + with diagnostics, and to return the model state at the next timestep. - Raises - ------ - SharedKeyError - If two components compute the same diagnostic quantity. - """ - if self.component_class is not None: - ensure_components_have_class(args, self.component_class) - self._components = args - if hasattr(self, 'diagnostics'): - if (len(self.diagnostics) != - sum([len(comp.diagnostics) for comp in self._components])): - raise SharedKeyError( - 'Two components in a composite should not compute ' - 'the same diagnostic') - - def _combine_attribute(self, attr): - return_attr = [] - for component in self._components: - return_attr.extend(getattr(component, attr)) - return tuple(set(return_attr)) # set to deduplicate - - -def ensure_components_have_class(components, component_class): - for component in components: - for attr in ('input_properties', 'output_properties', - 'diagnostic_properties', 'tendency_properties'): - if hasattr(component_class, attr) and not hasattr(component, attr): - raise TypeError( - 'Component should have attribute {} but does not'.format( - attr)) - elif hasattr(component, attr) and not hasattr(component_class, attr): - raise TypeError( - 'Component should not have attribute {}, but does'.format( - attr)) - - -class PrognosticComposite(ComponentComposite): - """ Attributes ---------- inputs : tuple of str The quantities required in the state when the object is called. - tendencies : tuple of str - The quantities for which tendencies are returned when - the object is called. - diagnostics : tuple of str - The diagnostic quantities returned when the object is called. + diagnostics: tuple of str + The quantities for which values for the old state are returned + when the object is called. + outputs: tuple of str + The quantities for which values for the new state are returned + when the object is called. """ - component_class = Prognostic - - def __call__(self, state): - """ - Gets tendencies and diagnostics from the passed model state. - - Args - ---- - state : dict - A model state dictionary. - - Returns - ------- - tendencies : dict - A dictionary whose keys are strings indicating - state quantities and values are the time derivative of those - quantities in units/second at the time of the input state. - diagnostics : dict - A dictionary whose keys are strings indicating - state quantities and values are the value of those quantities - at the time of the input state. - - Raises - ------ - SharedKeyError - If multiple Prognostic objects contained in the - collection return the same diagnostic quantity. - KeyError - If a required quantity is missing from the state. - InvalidStateError - If state is not a valid input for a Prognostic instance. - """ - return_tendencies = {} - return_diagnostics = {} - for prognostic in self._components: - tendencies, diagnostics = prognostic(state) - update_dict_by_adding_another(return_tendencies, tendencies) - return_diagnostics.update(diagnostics) - return return_tendencies, return_diagnostics - - @property - def inputs(self): - return self._combine_attribute('inputs') - - @property - def diagnostics(self): - return self._combine_attribute('diagnostics') - - @property - def tendencies(self): - return self._combine_attribute('tendencies') + __metaclass__ = abc.ABCMeta + def __str__(self): + return ( + 'instance of {}(TimeStepper)\n' + ' inputs: {}\n' + ' outputs: {}\n' + ' diagnostics: {}\n' + ' Prognostic components: {}'.format( + self.__class__, self.inputs, self.outputs, self.diagnostics, + str(self._prognostic)) + ) -class DiagnosticComposite(ComponentComposite): - """ - Attributes - ---------- - inputs : tuple of str - The quantities required in the state when the object is called. - diagnostics : tuple of str - The diagnostic quantities returned when the object is called. - """ + def __repr__(self): + if hasattr(self, '_making_repr') and self._making_repr: + return '{}(recursive reference)'.format(self.__class__) + else: + self._making_repr = True + return_value = '{}({})'.format( + self.__class__, + '\n'.join('{}: {}'.format(repr(key), repr(value)) + for key, value in self.__dict__.items() + if key != '_making_repr')) + self._making_repr = False + return return_value - component_class = Diagnostic + def __init__(self, prognostic_list, **kwargs): + self._prognostic = PrognosticComposite(*prognostic_list) - def __call__(self, state): + @abc.abstractmethod + def __call__(self, state, timestep): """ - Gets diagnostics from the passed model state. + Retrieves any diagnostics and returns a new state corresponding + to the next timestep. Args ---- state : dict - A model state dictionary. + The current model state. + timestep : timedelta + The amount of time to step forward. Returns ------- - diagnostics: dict - A dictionary whose keys are strings indicating - state quantities and values are the value of those quantities - at the time of the input state. - - Raises - ------ - SharedKeyError - If multiple Diagnostic objects contained in the - collection return the same diagnostic quantity. - KeyError - If a required quantity is missing from the state. - InvalidStateError - If state is not a valid input for a Diagnostic instance. + diagnostics : dict + Diagnostics from the timestep of the input state. + new_state : dict + The model state at the next timestep. """ - return_diagnostics = {} - for diagnostic_component in self._components: - diagnostics = diagnostic_component(state) - # ensure two diagnostics don't compute the same quantity - ensure_no_shared_keys(return_diagnostics, diagnostics) - return_diagnostics.update(diagnostics) - return return_diagnostics + + def _copy_untouched_quantities(self, old_state, new_state): + for key in old_state.keys(): + if key not in new_state: + new_state[key] = old_state[key] @property def inputs(self): - return self._combine_attribute('inputs') + return self._prognostic.inputs @property - def diagnostics(self): - return self._combine_attribute('diagnostics') - - -class MonitorComposite(ComponentComposite): - - component_class = Monitor - - def store(self, state): - """ - Stores the given state in the Monitor and performs class-specific - actions. - - Args - ---- - state : dict - A model state dictionary. + def outputs(self): + return self._prognostic.tendencies - Raises - ------ - KeyError - If a required quantity is missing from the state. - InvalidStateError - If state is not a valid input for a Monitor instance. - """ - for monitor in self._components: - monitor.store(state) + @property + def diagnostics(self): + return self._prognostic.diagnostics diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py new file mode 100644 index 0000000..ab4d15e --- /dev/null +++ b/sympl/_core/composite.py @@ -0,0 +1,206 @@ +from sympl import SharedKeyError, Prognostic, Diagnostic, ensure_no_shared_keys, \ + Monitor +from sympl._core.util import update_dict_by_adding_another + + +class ComponentComposite(object): + + component_class = None + + def __str__(self): + return '{}(\n{}\n)'.format( + self.__class__, + ',\n'.join(str(component) for component in self._components)) + + def __repr__(self): + return '{}(\n{}\n)'.format( + self.__class__, + ',\n'.join(repr(component) for component in self._components)) + + def __init__(self, *args): + """ + Args + ---- + *args + The components that should be wrapped by this object. + + Raises + ------ + SharedKeyError + If two components compute the same diagnostic quantity. + """ + if self.component_class is not None: + ensure_components_have_class(args, self.component_class) + self._components = args + if hasattr(self, 'diagnostics'): + if (len(self.diagnostics) != + sum([len(comp.diagnostics) for comp in self._components])): + raise SharedKeyError( + 'Two components in a composite should not compute ' + 'the same diagnostic') + + def _combine_attribute(self, attr): + return_attr = [] + for component in self._components: + return_attr.extend(getattr(component, attr)) + return tuple(set(return_attr)) # set to deduplicate + + +def ensure_components_have_class(components, component_class): + for component in components: + for attr in ('input_properties', 'output_properties', + 'diagnostic_properties', 'tendency_properties'): + if hasattr(component_class, attr) and not hasattr(component, attr): + raise TypeError( + 'Component should have attribute {} but does not'.format( + attr)) + elif hasattr(component, attr) and not hasattr(component_class, attr): + raise TypeError( + 'Component should not have attribute {}, but does'.format( + attr)) + + +class PrognosticComposite(ComponentComposite): + """ + Attributes + ---------- + inputs : tuple of str + The quantities required in the state when the object is called. + tendencies : tuple of str + The quantities for which tendencies are returned when + the object is called. + diagnostics : tuple of str + The diagnostic quantities returned when the object is called. + """ + + component_class = Prognostic + + def __call__(self, state): + """ + Gets tendencies and diagnostics from the passed model state. + + Args + ---- + state : dict + A model state dictionary. + + Returns + ------- + tendencies : dict + A dictionary whose keys are strings indicating + state quantities and values are the time derivative of those + quantities in units/second at the time of the input state. + diagnostics : dict + A dictionary whose keys are strings indicating + state quantities and values are the value of those quantities + at the time of the input state. + + Raises + ------ + SharedKeyError + If multiple Prognostic objects contained in the + collection return the same diagnostic quantity. + KeyError + If a required quantity is missing from the state. + InvalidStateError + If state is not a valid input for a Prognostic instance. + """ + return_tendencies = {} + return_diagnostics = {} + for prognostic in self._components: + tendencies, diagnostics = prognostic(state) + update_dict_by_adding_another(return_tendencies, tendencies) + return_diagnostics.update(diagnostics) + return return_tendencies, return_diagnostics + + @property + def inputs(self): + return self._combine_attribute('inputs') + + @property + def diagnostics(self): + return self._combine_attribute('diagnostics') + + @property + def tendencies(self): + return self._combine_attribute('tendencies') + + +class DiagnosticComposite(ComponentComposite): + """ + Attributes + ---------- + inputs : tuple of str + The quantities required in the state when the object is called. + diagnostics : tuple of str + The diagnostic quantities returned when the object is called. + """ + + component_class = Diagnostic + + def __call__(self, state): + """ + Gets diagnostics from the passed model state. + + Args + ---- + state : dict + A model state dictionary. + + Returns + ------- + diagnostics: dict + A dictionary whose keys are strings indicating + state quantities and values are the value of those quantities + at the time of the input state. + + Raises + ------ + SharedKeyError + If multiple Diagnostic objects contained in the + collection return the same diagnostic quantity. + KeyError + If a required quantity is missing from the state. + InvalidStateError + If state is not a valid input for a Diagnostic instance. + """ + return_diagnostics = {} + for diagnostic_component in self._components: + diagnostics = diagnostic_component(state) + # ensure two diagnostics don't compute the same quantity + ensure_no_shared_keys(return_diagnostics, diagnostics) + return_diagnostics.update(diagnostics) + return return_diagnostics + + @property + def inputs(self): + return self._combine_attribute('inputs') + + @property + def diagnostics(self): + return self._combine_attribute('diagnostics') + + +class MonitorComposite(ComponentComposite): + + component_class = Monitor + + def store(self, state): + """ + Stores the given state in the Monitor and performs class-specific + actions. + + Args + ---- + state : dict + A model state dictionary. + + Raises + ------ + KeyError + If a required quantity is missing from the state. + InvalidStateError + If state is not a valid input for a Monitor instance. + """ + for monitor in self._components: + monitor.store(state) From bce67d2e6c939456dae673f660e5da5158ed6193 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Mar 2018 13:34:20 -0700 Subject: [PATCH 02/98] Some stragglers from last commit --- sympl/__init__.py | 8 +++++--- sympl/_core/testing.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sympl/__init__.py b/sympl/__init__.py index 037fc1d..6ad8c49 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- from ._core.base_components import ( - Prognostic, Diagnostic, Implicit, Monitor, PrognosticComposite, - DiagnosticComposite, MonitorComposite, ImplicitPrognostic + Prognostic, Diagnostic, Implicit, Monitor, DiagnosticComposite, MonitorComposite, ImplicitPrognostic ) -from ._core.timestepping import TimeStepper, Leapfrog, AdamsBashforth +from sympl._core.composite import PrognosticComposite, DiagnosticComposite, \ + MonitorComposite +from sympl._core.base_components import TimeStepper +from sympl._components.timesteppers import AdamsBashforth, Leapfrog from ._core.exceptions import ( InvalidStateError, SharedKeyError, DependencyError, InvalidPropertyDictError) diff --git a/sympl/_core/testing.py b/sympl/_core/testing.py index 6df1a0d..af259ac 100644 --- a/sympl/_core/testing.py +++ b/sympl/_core/testing.py @@ -4,7 +4,7 @@ import xarray as xr from .util import same_list from .base_components import Diagnostic, Prognostic, Implicit -from .timestepping import TimeStepper +from sympl._core.base_components import TimeStepper import numpy as np from .units import is_valid_unit from datetime import timedelta From 7ecc2fc2c474c9f527e22d45a94c8a2b1a1621f5 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Mar 2018 14:29:56 -0700 Subject: [PATCH 03/98] Removed inputs/outputs/diagnostics from component API Also added tests for SSPRungeKutta and made (most of them) pass. Some tests are failing currently due to long-standing lack of proper implementation of *_properties dictionaries for time steppers. --- sympl/__init__.py | 8 +- sympl/_components/timesteppers.py | 27 ++++-- sympl/_core/base_components.py | 156 ------------------------------ sympl/_core/composite.py | 6 +- sympl/_core/testing.py | 12 +-- tests/test_base_components.py | 30 +++--- tests/test_timestepping.py | 25 +++-- 7 files changed, 64 insertions(+), 200 deletions(-) diff --git a/sympl/__init__.py b/sympl/__init__.py index 6ad8c49..4a5f6a0 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- from ._core.base_components import ( - Prognostic, Diagnostic, Implicit, Monitor, DiagnosticComposite, MonitorComposite, ImplicitPrognostic + Prognostic, Diagnostic, Implicit, Monitor, ImplicitPrognostic ) -from sympl._core.composite import PrognosticComposite, DiagnosticComposite, \ +from ._core.composite import PrognosticComposite, DiagnosticComposite, \ MonitorComposite -from sympl._core.base_components import TimeStepper -from sympl._components.timesteppers import AdamsBashforth, Leapfrog +from ._core.timestepper import TimeStepper +from ._components.timesteppers import AdamsBashforth, Leapfrog, SSPRungeKutta from ._core.exceptions import ( InvalidStateError, SharedKeyError, DependencyError, InvalidPropertyDictError) diff --git a/sympl/_components/timesteppers.py b/sympl/_components/timesteppers.py index d529031..e17924e 100644 --- a/sympl/_components/timesteppers.py +++ b/sympl/_components/timesteppers.py @@ -1,4 +1,4 @@ -from sympl._core.base_components import TimeStepper +from sympl._core.timestepper import TimeStepper from .._core.array import DataArray @@ -59,30 +59,39 @@ def _step_3_stages(self, state, timestep): state_2 = add(multiply(0.75, state), multiply(0.25, state_1_5)) _, state_2_5 = self._euler_stepper(state_2, timestep) out_state = add(multiply(1./3, state), multiply(2./3, state_2_5)) - return out_state + return diagnostics, out_state def _step_2_stages(self, state, timestep): + assert state is not None diagnostics, state_1 = self._euler_stepper(state, timestep) + assert state_1 is not None _, state_2 = self._euler_stepper(state_1, timestep) out_state = multiply(0.5, add(state, state_2)) - return out_state + return diagnostics, out_state def add(state_1, state_2): - out_state = {'time': state_1['time']} - for key in state.keys(): + out_state = {} + if 'time' in state_1.keys(): + out_state['time'] = state_1['time'] + for key in state_1.keys(): if key != 'time': - out_state[key] = state[key] + state_2[key] - out_state[key].attrs = state_1[key].attrs + out_state[key] = state_1[key] + state_2[key] + if hasattr(out_state[key], 'attrs'): + out_state[key].attrs = state_1[key].attrs + return out_state def multiply(scalar, state): - out_state = {'time': state['time']} + out_state = {} + if 'time' in state.keys(): + out_state['time'] = state['time'] for key in state.keys(): if key != 'time': out_state[key] = scalar * state[key] - out_state[key].attrs = state[key].attrs + if hasattr(out_state[key], 'attrs'): + out_state[key].attrs = state[key].attrs return out_state diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 7de58cc..a1ace91 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -1,20 +1,10 @@ import abc -from sympl._core.composite import PrognosticComposite - class Implicit(object): """ Attributes ---------- - inputs : tuple of str - The quantities required in the state when the object is called. - diagnostics: tuple of str - The quantities for which values for the old state are returned - when the object is called. - outputs: tuple of str - The quantities for which values for the new state are returned - when the object is called. input_properties : dict A dictionary whose keys are quantities required in the state when the object is called, and values are dictionaries which indicate 'dims' and @@ -36,18 +26,6 @@ class Implicit(object): diagnostic_properties = {} output_properties = {} - @property - def inputs(self): - return list(self.input_properties.keys()) - - @property - def diagnostics(self): - return list(self.diagnostic_properties.keys()) - - @property - def outputs(self): - return list(self.output_properties.keys()) - def __str__(self): return ( 'instance of {}(Implicit)\n' @@ -108,13 +86,6 @@ class Prognostic(object): """ Attributes ---------- - inputs : tuple of str - The quantities required in the state when the object is called. - tendencies : tuple of str - The quantities for which tendencies are returned when - the object is called. - diagnostics : tuple of str - The diagnostic quantities returned when the object is called. input_properties : dict A dictionary whose keys are quantities required in the state when the object is called, and values are dictionaries which indicate 'dims' and @@ -134,18 +105,6 @@ class Prognostic(object): tendency_properties = {} diagnostic_properties = {} - @property - def inputs(self): - return list(self.input_properties.keys()) - - @property - def tendencies(self): - return list(self.tendency_properties.keys()) - - @property - def diagnostics(self): - return list(self.diagnostic_properties.keys()) - def __str__(self): return ( 'instance of {}(Prognostic)\n' @@ -203,13 +162,6 @@ class ImplicitPrognostic(object): """ Attributes ---------- - inputs : tuple of str - The quantities required in the state when the object is called. - tendencies : tuple of str - The quantities for which tendencies are returned when - the object is called. - diagnostics : tuple of str - The diagnostic quantities returned when the object is called. input_properties : dict A dictionary whose keys are quantities required in the state when the object is called, and values are dictionaries which indicate 'dims' and @@ -229,18 +181,6 @@ class ImplicitPrognostic(object): tendency_properties = {} diagnostic_properties = {} - @property - def inputs(self): - return list(self.input_properties.keys()) - - @property - def tendencies(self): - return list(self.tendency_properties.keys()) - - @property - def diagnostics(self): - return list(self.diagnostic_properties.keys()) - def __str__(self): return ( 'instance of {}(Prognostic)\n' @@ -300,10 +240,6 @@ class Diagnostic(object): """ Attributes ---------- - inputs : tuple of str - The quantities required in the state when the object is called. - diagnostics : tuple of str - The diagnostic quantities returned when the object is called. input_properties : dict A dictionary whose keys are quantities required in the state when the object is called, and values are dictionaries which indicate 'dims' and @@ -318,14 +254,6 @@ class Diagnostic(object): input_properties = {} diagnostic_properties = {} - @property - def inputs(self): - return list(self.input_properties.keys()) - - @property - def diagnostics(self): - return list(self.diagnostic_properties.keys()) - def __str__(self): return ( 'instance of {}(Diagnostic)\n' @@ -410,87 +338,3 @@ def store(self, state): """ -class TimeStepper(object): - """An object which integrates model state forward in time. - - It uses Prognostic and Diagnostic objects to update the current model state - with diagnostics, and to return the model state at the next timestep. - - Attributes - ---------- - inputs : tuple of str - The quantities required in the state when the object is called. - diagnostics: tuple of str - The quantities for which values for the old state are returned - when the object is called. - outputs: tuple of str - The quantities for which values for the new state are returned - when the object is called. - """ - - __metaclass__ = abc.ABCMeta - - def __str__(self): - return ( - 'instance of {}(TimeStepper)\n' - ' inputs: {}\n' - ' outputs: {}\n' - ' diagnostics: {}\n' - ' Prognostic components: {}'.format( - self.__class__, self.inputs, self.outputs, self.diagnostics, - str(self._prognostic)) - ) - - def __repr__(self): - if hasattr(self, '_making_repr') and self._making_repr: - return '{}(recursive reference)'.format(self.__class__) - else: - self._making_repr = True - return_value = '{}({})'.format( - self.__class__, - '\n'.join('{}: {}'.format(repr(key), repr(value)) - for key, value in self.__dict__.items() - if key != '_making_repr')) - self._making_repr = False - return return_value - - def __init__(self, prognostic_list, **kwargs): - self._prognostic = PrognosticComposite(*prognostic_list) - - @abc.abstractmethod - def __call__(self, state, timestep): - """ - Retrieves any diagnostics and returns a new state corresponding - to the next timestep. - - Args - ---- - state : dict - The current model state. - timestep : timedelta - The amount of time to step forward. - - Returns - ------- - diagnostics : dict - Diagnostics from the timestep of the input state. - new_state : dict - The model state at the next timestep. - """ - - def _copy_untouched_quantities(self, old_state, new_state): - for key in old_state.keys(): - if key not in new_state: - new_state[key] = old_state[key] - - @property - def inputs(self): - return self._prognostic.inputs - - @property - def outputs(self): - return self._prognostic.tendencies - - @property - def diagnostics(self): - return self._prognostic.diagnostics diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index ab4d15e..9d679c7 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -1,6 +1,6 @@ -from sympl import SharedKeyError, Prognostic, Diagnostic, ensure_no_shared_keys, \ - Monitor -from sympl._core.util import update_dict_by_adding_another +from .exceptions import SharedKeyError +from .base_components import Prognostic, Diagnostic, Monitor +from sympl._core.util import update_dict_by_adding_another, ensure_no_shared_keys class ComponentComposite(object): diff --git a/sympl/_core/testing.py b/sympl/_core/testing.py index af259ac..f2b3091 100644 --- a/sympl/_core/testing.py +++ b/sympl/_core/testing.py @@ -4,7 +4,7 @@ import xarray as xr from .util import same_list from .base_components import Diagnostic, Prognostic, Implicit -from sympl._core.base_components import TimeStepper +from sympl._core.timestepper import TimeStepper import numpy as np from .units import is_valid_unit from datetime import timedelta @@ -151,15 +151,15 @@ def test_listed_outputs_are_accurate(self): component = self.get_component_instance() if isinstance(component, Diagnostic): diagnostics = component(state) - assert same_list(component.diagnostics, diagnostics.keys()) + assert same_list(component.diagnostic_properties.keys(), diagnostics.keys()) elif isinstance(component, Prognostic): tendencies, diagnostics = component(state) - assert same_list(component.tendencies, tendencies.keys()) - assert same_list(component.diagnostics, diagnostics.keys()) + assert same_list(component.tendency_properties.keys(), tendencies.keys()) + assert same_list(component.diagnostic_properties.keys(), diagnostics.keys()) elif isinstance(component, Implicit): diagnostics, new_state = component(state) - assert same_list(component.diagnostics, diagnostics.keys()) - assert same_list(component.outputs, new_state.keys()) + assert same_list(component.diagnostic_properties.keys(), diagnostics.keys()) + assert same_list(component.output_properties.keys(), new_state.keys()) def test_modifies_attribute_is_accurate(self): state = self.get_input_state() diff --git a/tests/test_base_components.py b/tests/test_base_components.py index 386e6d5..f030814 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -152,9 +152,9 @@ def test_prognostic_composite_includes_attributes(): prognostic.diagnostic_properties = {'diagnostic1': {'units': 'm/s'}} prognostic.tendency_properties = {'tendency1': {}} composite = PrognosticComposite(prognostic) - assert composite.inputs == ('input1',) - assert composite.diagnostics == ('diagnostic1',) - assert composite.tendencies == ('tendency1',) + assert same_list(composite.input_properties.keys(), ('input1',)) + assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1',)) + assert same_list(composite.tendency_properties.keys(), ('tendency1',)) def test_prognostic_composite_includes_attributes_from_two(): @@ -167,9 +167,9 @@ def test_prognostic_composite_includes_attributes_from_two(): prognostic2.diagnostic_properties = {'diagnostic2': {'units': 'm/s'}} prognostic2.tendency_properties = {'tendency2': {}} composite = PrognosticComposite(prognostic1, prognostic2) - assert same_list(composite.inputs, ('input1', 'input2')) - assert same_list(composite.diagnostics, ('diagnostic1', 'diagnostic2')) - assert same_list(composite.tendencies, ('tendency1', 'tendency2')) + assert same_list(composite.input_properties.keys(), ('input1', 'input2')) + assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) + assert same_list(composite.tendency_properties.keys(), ('tendency1', 'tendency2')) def test_prognostic_merges_attributes(): @@ -182,9 +182,9 @@ def test_prognostic_merges_attributes(): prognostic2.diagnostic_properties = {'diagnostic2': {}} prognostic2.tendency_properties = {'tendency2': {}} composite = PrognosticComposite(prognostic1, prognostic2) - assert same_list(composite.inputs, ('input1', 'input2')) - assert same_list(composite.diagnostics, ('diagnostic1', 'diagnostic2')) - assert same_list(composite.tendencies, ('tendency1', 'tendency2')) + assert same_list(composite.input_properties.keys(), ('input1', 'input2')) + assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) + assert same_list(composite.tendency_properties.keys(), ('tendency1', 'tendency2')) def test_prognostic_composite_ensures_valid_state(): @@ -224,8 +224,8 @@ def test_diagnostic_composite_includes_attributes(): diagnostic.input_properties = {'input1': {}} diagnostic.diagnostic_properties = {'diagnostic1': {}} composite = DiagnosticComposite(diagnostic) - assert composite.inputs == ('input1',) - assert composite.diagnostics == ('diagnostic1',) + assert same_list(composite.input_properties.keys(), ('input1',)) + assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1',)) def test_diagnostic_composite_includes_attributes_from_two(): @@ -236,8 +236,8 @@ def test_diagnostic_composite_includes_attributes_from_two(): diagnostic2.input_properties = {'input2': {}} diagnostic2.diagnostic_properties = {'diagnostic2': {}} composite = DiagnosticComposite(diagnostic1, diagnostic2) - assert same_list(composite.inputs, ('input1', 'input2')) - assert same_list(composite.diagnostics, ('diagnostic1', 'diagnostic2')) + assert same_list(composite.input_properties.keys(), ('input1', 'input2')) + assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) def test_diagnostic_composite_merges_attributes(): @@ -248,8 +248,8 @@ def test_diagnostic_composite_merges_attributes(): diagnostic2.input_properties = {'input1': {}, 'input2': {}} diagnostic2.diagnostic_properties = {'diagnostic2': {}} composite = DiagnosticComposite(diagnostic1, diagnostic2) - assert same_list(composite.inputs, ('input1', 'input2')) - assert same_list(composite.diagnostics, ('diagnostic1', 'diagnostic2')) + assert same_list(composite.input_properties.keys(), ('input1', 'input2')) + assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) def test_diagnostic_composite_ensures_valid_state(): diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index f4416e3..388cde9 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -1,7 +1,7 @@ import pytest import mock from sympl import ( - Prognostic, Leapfrog, AdamsBashforth, DataArray) + Prognostic, Leapfrog, AdamsBashforth, DataArray, SSPRungeKutta) from datetime import timedelta import numpy as np @@ -33,7 +33,7 @@ def test_timestepper_reveals_inputs(self): prog1 = MockPrognostic() prog1.input_properties = {'input1': {}} time_stepper = self.timestepper_class([prog1]) - assert same_list(time_stepper.inputs, ('input1',)) + assert same_list(time_stepper.input_properties.keys(), ('input1',)) def test_timestepper_combines_inputs(self): prog1 = MockPrognostic() @@ -41,7 +41,7 @@ def test_timestepper_combines_inputs(self): prog2 = MockPrognostic() prog2.input_properties = {'input2': {}} time_stepper = self.timestepper_class([prog1, prog2]) - assert same_list(time_stepper.inputs, ('input1', 'input2')) + assert same_list(time_stepper.input_properties.keys(), ('input1', 'input2')) def test_timestepper_doesnt_duplicate_inputs(self): prog1 = MockPrognostic() @@ -49,13 +49,13 @@ def test_timestepper_doesnt_duplicate_inputs(self): prog2 = MockPrognostic() prog2.input_properties = {'input1': {}} time_stepper = self.timestepper_class([prog1, prog2]) - assert same_list(time_stepper.inputs, ('input1',)) + assert same_list(time_stepper.input_properties.keys(), ('input1',)) def test_timestepper_reveals_outputs(self): prog1 = MockPrognostic() prog1.tendency_properties = {'output1': {}} time_stepper = self.timestepper_class([prog1]) - assert same_list(time_stepper.outputs, ('output1',)) + assert same_list(time_stepper.output_properties.keys(), ('output1',)) def test_timestepper_combines_outputs(self): prog1 = MockPrognostic() @@ -63,7 +63,7 @@ def test_timestepper_combines_outputs(self): prog2 = MockPrognostic() prog2.tendency_properties = {'output2': {}} time_stepper = self.timestepper_class([prog1, prog2]) - assert same_list(time_stepper.outputs, ('output1', 'output2')) + assert same_list(time_stepper.output_properties.keys(), ('output1', 'output2')) def test_timestepper_doesnt_duplicate_outputs(self): prog1 = MockPrognostic() @@ -71,7 +71,7 @@ def test_timestepper_doesnt_duplicate_outputs(self): prog2 = MockPrognostic() prog2.tendency_properties = {'output1': {}} time_stepper = self.timestepper_class([prog1, prog2]) - assert same_list(time_stepper.outputs, ('output1',)) + assert same_list(time_stepper.output_properties.keys(), ('output1',)) @mock.patch.object(MockPrognostic, '__call__') def test_float_no_change_one_step(self, mock_prognostic_call): @@ -341,6 +341,17 @@ def test_dataarray_three_steps(self, mock_prognostic_call): assert new_state['air_temperature'].attrs['units'] == 'K' +class TestSSPRungeKuttaTwoStep(TimesteppingBase): + + def timestepper_class(self, *args): + return SSPRungeKutta(*args, stages=2) + + +class TestSSPRungeKuttaThreeStep(TimesteppingBase): + def timestepper_class(self, *args): + return SSPRungeKutta(*args, stages=3) + + class TestLeapfrog(TimesteppingBase): timestepper_class = Leapfrog From ad70996252ee982c2e2a07907287ad8b5d24f4b1 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Mar 2018 14:32:24 -0700 Subject: [PATCH 04/98] Added history note about component API change --- HISTORY.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8581594..a437392 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,7 +5,11 @@ What's New Latest ------ -* No changes yet! +Breaking changes +~~~~~~~~~~~~~~~~ + +* inputs, outputs, diagnotsics, and tendencies are no longer attributes of components. + In order to get these, you should use e.g. input_properties.keys() v0.3.1 ------ From 6118e743007355e09a98debdd560c574b27433c1 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 16 Mar 2018 16:49:45 -0700 Subject: [PATCH 05/98] Changed the way component subclasses should be written. * properties dictionaries are now abstract methods, so subclasses must define them. Previously they defaulted to empty dictionaries. * Components should now be written using a new array_call method rather than __call__. __call__ will automatically unwrap DataArrays to numpy arrays to be passed into array_call based on the component's properties dictionaries, and re-wrap to DataArrays when done. --- HISTORY.rst | 6 + sympl/_components/timesteppers.py | 31 +---- sympl/_core/base_components.py | 189 +++++++++++++++++++++++++++--- sympl/_core/state.py | 29 +++++ sympl/_core/timestepper.py | 92 +++++++++++++++ 5 files changed, 302 insertions(+), 45 deletions(-) create mode 100644 sympl/_core/state.py create mode 100644 sympl/_core/timestepper.py diff --git a/HISTORY.rst b/HISTORY.rst index a437392..943b933 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,12 @@ Breaking changes * inputs, outputs, diagnotsics, and tendencies are no longer attributes of components. In order to get these, you should use e.g. input_properties.keys() +* properties dictionaries are now abstract methods, so subclasses must define them. + Previously they defaulted to empty dictionaries. +* Components should now be written using a new array_call method rather than __call__. + __call__ will automatically unwrap DataArrays to numpy arrays to be passed into + array_call based on the component's properties dictionaries, and re-wrap to + DataArrays when done. v0.3.1 ------ diff --git a/sympl/_components/timesteppers.py b/sympl/_components/timesteppers.py index e17924e..955bcc7 100644 --- a/sympl/_components/timesteppers.py +++ b/sympl/_components/timesteppers.py @@ -1,5 +1,6 @@ -from sympl._core.timestepper import TimeStepper +from .._core.timestepper import TimeStepper from .._core.array import DataArray +from .._core.state import copy_untouched_quantities, add, multiply class SSPRungeKutta(TimeStepper): @@ -71,30 +72,6 @@ def _step_2_stages(self, state, timestep): return diagnostics, out_state -def add(state_1, state_2): - out_state = {} - if 'time' in state_1.keys(): - out_state['time'] = state_1['time'] - for key in state_1.keys(): - if key != 'time': - out_state[key] = state_1[key] + state_2[key] - if hasattr(out_state[key], 'attrs'): - out_state[key].attrs = state_1[key].attrs - return out_state - - -def multiply(scalar, state): - out_state = {} - if 'time' in state.keys(): - out_state['time'] = state['time'] - for key in state.keys(): - if key != 'time': - out_state[key] = scalar * state[key] - if hasattr(out_state[key], 'attrs'): - out_state[key].attrs = state[key].attrs - return out_state - - class AdamsBashforth(TimeStepper): """A TimeStepper using the Adams-Bashforth scheme.""" @@ -153,7 +130,7 @@ def __call__(self, state, timestep): convert_tendencies_units_for_state(tendencies, state) self._tendencies_list.append(tendencies) new_state = self._perform_step(state, timestep) - self._copy_untouched_quantities(state, new_state) + copy_untouched_quantities(state, new_state) if len(self._tendencies_list) == self._order: self._tendencies_list.pop(0) # remove the oldest entry return diagnostics, new_state @@ -280,7 +257,7 @@ def __call__(self, state, timestep): state, new_state = step_leapfrog( self._old_state, state, tendencies, timestep, asselin_strength=self._asselin_strength, alpha=self._alpha) - self._copy_untouched_quantities(state, new_state) + copy_untouched_quantities(state, new_state) self._old_state = state for key in original_state.keys(): original_state[key] = state[key] # allow filtering to be applied diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index a1ace91..92e6677 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -1,4 +1,6 @@ import abc +from util import ( + get_numpy_arrays_with_properties, restore_data_arrays_with_properties) class Implicit(object): @@ -22,9 +24,17 @@ class Implicit(object): """ __metaclass__ = abc.ABCMeta - input_properties = {} - diagnostic_properties = {} - output_properties = {} + @abc.abstractproperty + def input_properties(self): + return {} + + @abc.abstractproperty + def diagnostic_properties(self): + return {} + + @abc.abstractproperty + def output_properties(self): + return {} def __str__(self): return ( @@ -48,7 +58,6 @@ def __repr__(self): self._making_repr = False return return_value - @abc.abstractmethod def __call__(self, state, timestep): """ Gets diagnostics from the current model state and steps the state @@ -57,9 +66,7 @@ def __call__(self, state, timestep): Args ---- state : dict - A model state dictionary. Will be updated with any - diagnostic quantities produced by this object for the time of - the input state. + A model state dictionary. timestep : timedelta The amount of time to step forward. @@ -80,6 +87,40 @@ def __call__(self, state, timestep): If state is not a valid input for the Implicit instance for other reasons. """ + state = get_numpy_arrays_with_properties(state, self.input_properties) + raw_diagnostics, raw_new_state = self.array_call(state, timestep) + diagnostics = restore_data_arrays_with_properties( + raw_diagnostics, self.diagnostic_properties) + new_state = restore_data_arrays_with_properties( + raw_new_state, self.output_properties) + return diagnostics, new_state + + + @abc.abstractmethod + def array_call(self, state, timestep): + """ + Gets diagnostics from the current model state and steps the state + forward in time according to the timestep. + + Args + ---- + state : dict + A numpy array state dictionary. Instead of data arrays, should + include numpy arrays that satisfy the input properties of this + object. + timestep : timedelta + The amount of time to step forward. + + Returns + ------- + diagnostics : dict + Diagnostics from the timestep of the input state, as numpy arrays. + new_state : dict + A dictionary whose keys are strings indicating + state quantities and values are the value of those quantities + at the timestep after input state, as numpy arrays. + """ + pass class Prognostic(object): @@ -101,9 +142,17 @@ class Prognostic(object): """ __metaclass__ = abc.ABCMeta - input_properties = {} - tendency_properties = {} - diagnostic_properties = {} + @abc.abstractproperty + def input_properties(self): + return {} + + @abc.abstractproperty + def tendency_properties(self): + return {} + + @abc.abstractproperty + def diagnostic_properties(self): + return {} def __str__(self): return ( @@ -127,7 +176,6 @@ def __repr__(self): self._making_repr = False return return_value - @abc.abstractmethod def __call__(self, state): """ Gets tendencies and diagnostics from the passed model state. @@ -156,6 +204,40 @@ def __call__(self, state): InvalidStateError If state is not a valid input for the Prognostic instance. """ + raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + raw_tendencies, raw_diagnostics = self.array_call(raw_state) + tendencies = restore_data_arrays_with_properties( + raw_tendencies, self.tendency_properties) + diagnostics = restore_data_arrays_with_properties( + raw_diagnostics, self.diagnostic_properties) + return tendencies, diagnostics + + @abc.abstractmethod + def array_call(self, state): + """ + Gets tendencies and diagnostics from the passed model state. + + Args + ---- + state : dict + A model state dictionary. Instead of data arrays, should + include numpy arrays that satisfy the input properties of this + object. + + Returns + ------- + tendencies : dict + A dictionary whose keys are strings indicating + state quantities and values are the time derivative of those + quantities in units/second at the time of the input state, as + numpy arrays. + + diagnostics : dict + A dictionary whose keys are strings indicating + state quantities and values are the value of those quantities + at the time of the input state, as numpy arrays. + """ + pass class ImplicitPrognostic(object): @@ -177,9 +259,17 @@ class ImplicitPrognostic(object): """ __metaclass__ = abc.ABCMeta - input_properties = {} - tendency_properties = {} - diagnostic_properties = {} + @abc.abstractproperty + def input_properties(self): + return {} + + @abc.abstractproperty + def tendency_properties(self): + return {} + + @abc.abstractproperty + def diagnostic_properties(self): + return {} def __str__(self): return ( @@ -203,7 +293,6 @@ def __repr__(self): self._making_repr = False return return_value - @abc.abstractmethod def __call__(self, state, timestep): """ Gets tendencies and diagnostics from the passed model state. @@ -234,6 +323,41 @@ def __call__(self, state, timestep): InvalidStateError If state is not a valid input for the Prognostic instance. """ + raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + raw_tendencies, raw_diagnostics = self.array_call(raw_state, timestep) + tendencies = restore_data_arrays_with_properties( + raw_tendencies, self.tendency_properties) + diagnostics = restore_data_arrays_with_properties( + raw_diagnostics, self.diagnostic_properties) + return tendencies, diagnostics + + @abc.abstractmethod + def array_call(self, state, timestep): + """ + Gets tendencies and diagnostics from the passed model state. + + Args + ---- + state : dict + A model state dictionary. Instead of data arrays, should + include numpy arrays that satisfy the input properties of this + object. + timestep : timedelta + The time over which the model is being stepped. + + Returns + ------- + tendencies : dict + A dictionary whose keys are strings indicating + state quantities and values are the time derivative of those + quantities in units/second at the time of the input state, as + numpy arrays. + + diagnostics : dict + A dictionary whose keys are strings indicating + state quantities and values are the value of those quantities + at the time of the input state, as numpy arrays. + """ class Diagnostic(object): @@ -251,8 +375,13 @@ class Diagnostic(object): """ __metaclass__ = abc.ABCMeta - input_properties = {} - diagnostic_properties = {} + @abc.abstractproperty + def input_properties(self): + return {} + + @abc.abstractproperty + def diagnostic_properties(self): + return {} def __str__(self): return ( @@ -275,7 +404,6 @@ def __repr__(self): self._making_repr = False return return_value - @abc.abstractmethod def __call__(self, state): """ Gets diagnostics from the passed model state. @@ -299,6 +427,31 @@ def __call__(self, state): InvalidStateError If state is not a valid input for the Prognostic instance. """ + raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + raw_diagnostics = self.array_call(raw_state) + diagnostics = restore_data_arrays_with_properties( + raw_diagnostics, self.diagnostic_properties) + return diagnostics + + @abc.abstractmethod + def array_call(self, state): + """ + Gets diagnostics from the passed model state. + + Args + ---- + state : dict + A model state dictionary. Instead of data arrays, should + include numpy arrays that satisfy the input properties of this + object. + + Returns + ------- + diagnostics : dict + A dictionary whose keys are strings indicating + state quantities and values are the value of those quantities + at the time of the input state, as numpy arrays. + """ class Monitor(object): diff --git a/sympl/_core/state.py b/sympl/_core/state.py new file mode 100644 index 0000000..45fc154 --- /dev/null +++ b/sympl/_core/state.py @@ -0,0 +1,29 @@ + +def copy_untouched_quantities(old_state, new_state): + for key in old_state.keys(): + if key not in new_state: + new_state[key] = old_state[key] + + +def add(state_1, state_2): + out_state = {} + if 'time' in state_1.keys(): + out_state['time'] = state_1['time'] + for key in state_1.keys(): + if key != 'time': + out_state[key] = state_1[key] + state_2[key] + if hasattr(out_state[key], 'attrs'): + out_state[key].attrs = state_1[key].attrs + return out_state + + +def multiply(scalar, state): + out_state = {} + if 'time' in state.keys(): + out_state['time'] = state['time'] + for key in state.keys(): + if key != 'time': + out_state[key] = scalar * state[key] + if hasattr(out_state[key], 'attrs'): + out_state[key].attrs = state[key].attrs + return out_state diff --git a/sympl/_core/timestepper.py b/sympl/_core/timestepper.py new file mode 100644 index 0000000..ce6bd6c --- /dev/null +++ b/sympl/_core/timestepper.py @@ -0,0 +1,92 @@ +import abc + +from .composite import PrognosticComposite + + +class TimeStepper(object): + """An object which integrates model state forward in time. + + It uses Prognostic and Diagnostic objects to update the current model state + with diagnostics, and to return the model state at the next timestep. + + Attributes + ---------- + input_properties : dict + A dictionary whose keys are quantities required in the state when the + object is called, and values are dictionaries which indicate 'dims' and + 'units'. + diagnostic_properties : dict + A dictionary whose keys are quantities for which values + for the old state are returned when the + object is called, and values are dictionaries which indicate 'dims' and + 'units'. + output_properties : dict + A dictionary whose keys are quantities for which values + for the new state are returned when the + object is called, and values are dictionaries which indicate 'dims' and + 'units'. + """ + + __metaclass__ = abc.ABCMeta + + @abc.abstractproperty + def input_properties(self): + return {} + + @abc.abstractproperty + def diagnostic_properties(self): + return {} + + @abc.abstractproperty + def output_properties(self): + return {} + + def __str__(self): + return ( + 'instance of {}(TimeStepper)\n' + ' inputs: {}\n' + ' outputs: {}\n' + ' diagnostics: {}\n' + ' Prognostic components: {}'.format( + self.__class__, self.input_properties.keys(), + self.output_properties.keys(), + self.diagnostic_properties.keys(), + str(self._prognostic)) + ) + + def __repr__(self): + if hasattr(self, '_making_repr') and self._making_repr: + return '{}(recursive reference)'.format(self.__class__) + else: + self._making_repr = True + return_value = '{}({})'.format( + self.__class__, + '\n'.join('{}: {}'.format(repr(key), repr(value)) + for key, value in self.__dict__.items() + if key != '_making_repr')) + self._making_repr = False + return return_value + + def __init__(self, prognostic_list, **kwargs): + self._prognostic = PrognosticComposite(*prognostic_list) + + @abc.abstractmethod + def __call__(self, state, timestep): + """ + Retrieves any diagnostics and returns a new state corresponding + to the next timestep. + + Args + ---- + state : dict + The current model state. + timestep : timedelta + The amount of time to step forward. + + Returns + ------- + diagnostics : dict + Diagnostics from the timestep of the input state. + new_state : dict + The model state at the next timestep. + """ From d764743c925f90c2ebb91d795bbe2f988fca4fa5 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 19 Mar 2018 16:24:33 -0700 Subject: [PATCH 06/98] Refactored wrapper functionality into components * Functionality provided by ScalingWrapper, UpdateFrequencyWrapper, and TendencyInDiagnosticsWrapper is now provided by base components instead of wrappers. * inputs, outputs, diagnostics, and tendencies are no longer attributes of components * composites now have a composite_list attribute --- HISTORY.rst | 10 + sympl/_components/timesteppers.py | 4 +- sympl/_core/base_components.py | 403 +++++++++++++++++++++++++++--- sympl/_core/composite.py | 66 ++--- sympl/_core/timestepper.py | 117 +++++++-- sympl/_core/wrappers.py | 268 -------------------- 6 files changed, 496 insertions(+), 372 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 943b933..96ad794 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,13 @@ What's New Latest ------ +* Implicit, Diagnostic, ImplicitPrognostic, and Prognostic base classes were + modified to include functionality that was previously in ScalingWrapper, + UpdateFrequencyWrapper, and TendencyInDiagnosticsWrapper. The functionality of + TendencyInDiagnosticsWrapper is now to be used in Implicit and TimeStepper objects. +* Composites now have a component_list attribute which contains the components being + composited. + Breaking changes ~~~~~~~~~~~~~~~~ @@ -16,6 +23,9 @@ Breaking changes __call__ will automatically unwrap DataArrays to numpy arrays to be passed into array_call based on the component's properties dictionaries, and re-wrap to DataArrays when done. +* ScalingWrapper, UpdateFrequencyWrapper, and TendencyInDiagnosticsWrapper + have been removed. The functionality of these wrappers has been moved to the + component base types as methods and initialization options. v0.3.1 ------ diff --git a/sympl/_components/timesteppers.py b/sympl/_components/timesteppers.py index 955bcc7..6b08990 100644 --- a/sympl/_components/timesteppers.py +++ b/sympl/_components/timesteppers.py @@ -126,7 +126,7 @@ def __call__(self, state, timestep): """ self._ensure_constant_timestep(timestep) state = state.copy() - tendencies, diagnostics = self._prognostic(state) + tendencies, diagnostics = self.prognostic(state) convert_tendencies_units_for_state(tendencies, state) self._tendencies_list.append(tendencies) new_state = self._perform_step(state, timestep) @@ -249,7 +249,7 @@ def __call__(self, state, timestep): original_state = state state = state.copy() self._ensure_constant_timestep(timestep) - tendencies, diagnostics = self._prognostic(state) + tendencies, diagnostics = self.prognostic(state) convert_tendencies_units_for_state(tendencies, state) if self._old_state is None: new_state = step_forward_euler(state, tendencies, timestep) diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 92e6677..db4b8b7 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -1,6 +1,12 @@ import abc from util import ( get_numpy_arrays_with_properties, restore_data_arrays_with_properties) +from .time import timedelta + + +def apply_scale_factors(array_state, scale_factors): + for key, factor in scale_factors.items(): + array_state[key] *= factor class Implicit(object): @@ -21,9 +27,38 @@ class Implicit(object): for the new state are returned when the object is called, and values are dictionaries which indicate 'dims' and 'units'. + input_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + output_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which output values are scaled before being + returned by this object. + diagnostic_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + tendencies_in_diagnostics : bool + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output based on first order time + differencing of output values. + update_interval : timedelta + If not None, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. + time_unit_name : str + The unit to use for time differencing when putting tendencies in + diagnostics. + time_unit_timedelta: timedelta + A timedelta corresponding to a single time unit as used for time + differencing when putting tendencies in diagnostics. """ __metaclass__ = abc.ABCMeta + time_unit_name = 's' + time_unit_timedelta = timedelta(seconds=1) + @abc.abstractproperty def input_properties(self): return {} @@ -42,7 +77,9 @@ def __str__(self): ' inputs: {}\n' ' outputs: {}\n' ' diagnostics: {}'.format( - self.__class__, self.inputs, self.outputs, self.diagnostics) + self.__class__, self.input_properties.keys(), + self.output_properties.keys(), + self.diagnostic_properties.keys()) ) def __repr__(self): @@ -58,6 +95,81 @@ def __repr__(self): self._making_repr = False return return_value + def __init__(self, + input_scale_factors=None, output_scale_factors=None, + diagnostic_scale_factors=None, tendencies_in_diagnostics=False, + update_interval=None, name=None): + """ + Initializes the Implicit object. + + Args + ---- + input_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + output_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which output values are scaled before being + returned by this object. + diagnostic_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + tendencies_in_diagnostics : bool + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output based on first order time + differencing of output values. + update_interval : timedelta + If given, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. + name : string + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". By default the class name in + lowercase is used. + """ + if input_scale_factors is not None: + self.input_scale_factors = input_scale_factors + else: + self.input_scale_factors = {} + if output_scale_factors is not None: + self.output_scale_factors = output_scale_factors + else: + self.output_scale_factors = {} + if diagnostic_scale_factors is not None: + self.diagnostic_scale_factors = diagnostic_scale_factors + else: + self.diagnostic_scale_factors = {} + self._tendencies_in_diagnostics = tendencies_in_diagnostics + self.update_interval = update_interval + self._last_update_time = None + if self.name is None: + self.name = self.__class__.__name__.lower() + else: + self.name = name + if tendencies_in_diagnostics: + self._insert_tendencies_to_diagnostic_properties() + + def _insert_tendencies_to_diagnostic_properties(self): + for name, properties in self.output_properties.items(): + tendency_name = self._get_tendency_name(name) + if properties['units'] is '': + units = 's^-1' + else: + units = '{} s^-1'.format(properties['units']) + self.diagnostic_properties[tendency_name] = { + 'units': units, + 'dims': properties['dims'], + } + + def _get_tendency_name(self, name): + return '{}_tendency_from_{}'.format(name, self.name) + + @property + def tendencies_in_diagnostics(self): + return self._tendencies_in_diagnostics # value cannot be modified + def __call__(self, state, timestep): """ Gets diagnostics from the current model state and steps the state @@ -87,14 +199,31 @@ def __call__(self, state, timestep): If state is not a valid input for the Implicit instance for other reasons. """ - state = get_numpy_arrays_with_properties(state, self.input_properties) - raw_diagnostics, raw_new_state = self.array_call(state, timestep) - diagnostics = restore_data_arrays_with_properties( - raw_diagnostics, self.diagnostic_properties) - new_state = restore_data_arrays_with_properties( - raw_new_state, self.output_properties) - return diagnostics, new_state - + if (self.update_interval is None or + self._last_update_time is None or + state['time'] > self._last_update_time + self.update_interval): + raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + apply_scale_factors(raw_state, self.input_scale_factors) + raw_diagnostics, raw_new_state = self.array_call(state, timestep) + apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) + apply_scale_factors(raw_new_state, self.output_scale_factors) + if self.tendencies_in_diagnostics: + self._insert_tendencies_to_diagnostics( + raw_state, raw_new_state, timestep, raw_diagnostics) + self._diagnostics = restore_data_arrays_with_properties( + raw_diagnostics, self.diagnostic_properties) + self._new_state = restore_data_arrays_with_properties( + raw_new_state, self.output_properties) + self._last_update_time = state['time'] + return self._diagnostics, self._new_state + + def _insert_tendencies_to_diagnostics( + self, raw_state, raw_new_state, timestep, raw_diagnostics): + for name in self.output_properties.keys(): + tendency_name = self._get_tendency_name(name) + raw_diagnostics[tendency_name] = ( + (raw_new_state[name] - raw_state[name]) / + timestep.total_seconds() * self.time_unit_timedelta.total_seconds()) @abc.abstractmethod def array_call(self, state, timestep): @@ -139,6 +268,26 @@ class Prognostic(object): A dictionary whose keys are diagnostic quantities returned when the object is called, and values are dictionaries which indicate 'dims' and 'units'. + input_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + tendency_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which tendency values are scaled before being + returned by this object. + diagnostic_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + update_interval : timedelta + If not None, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. + name : string + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". By default the class name in + lowercase is used. """ __metaclass__ = abc.ABCMeta @@ -160,7 +309,9 @@ def __str__(self): ' inputs: {}\n' ' tendencies: {}\n' ' diagnostics: {}'.format( - self.__class__, self.inputs, self.tendencies, self.diagnostics) + self.__class__, self.input_properties.keys(), + self.tendency_properties.keys(), + self.diagnostic_properties.keys()) ) def __repr__(self): @@ -176,6 +327,54 @@ def __repr__(self): self._making_repr = False return return_value + def __init__(self, + input_scale_factors=None, tendency_scale_factors=None, + diagnostic_scale_factors=None, update_interval=None, name=None): + """ + Initializes the Implicit object. + + Args + ---- + input_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + tendency_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which tendency values are scaled before being + returned by this object. + diagnostic_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + update_interval : timedelta + If given, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. + name : string + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". By default the class name in + lowercase is used. + """ + if input_scale_factors is not None: + self.input_scale_factors = input_scale_factors + else: + self.input_scale_factors = {} + if tendency_scale_factors is not None: + self.tendency_scale_factors = tendency_scale_factors + else: + self.tendency_scale_factors = {} + if diagnostic_scale_factors is not None: + self.diagnostic_scale_factors = diagnostic_scale_factors + else: + self.diagnostic_scale_factors = {} + self.update_interval = update_interval + self._last_update_time = None + if self.name is None: + self.name = self.__class__.__name__.lower() + else: + self.name = name + def __call__(self, state): """ Gets tendencies and diagnostics from the passed model state. @@ -204,13 +403,20 @@ def __call__(self, state): InvalidStateError If state is not a valid input for the Prognostic instance. """ - raw_state = get_numpy_arrays_with_properties(state, self.input_properties) - raw_tendencies, raw_diagnostics = self.array_call(raw_state) - tendencies = restore_data_arrays_with_properties( - raw_tendencies, self.tendency_properties) - diagnostics = restore_data_arrays_with_properties( - raw_diagnostics, self.diagnostic_properties) - return tendencies, diagnostics + if (self.update_interval is None or + self._last_update_time is None or + state['time'] > self._last_update_time + self.update_interval): + raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + apply_scale_factors(raw_state, self.input_scale_factors) + raw_tendencies, raw_diagnostics = self.array_call(raw_state) + apply_scale_factors(raw_tendencies, self.tendency_scale_factors) + apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) + self._tendencies = restore_data_arrays_with_properties( + raw_tendencies, self.tendency_properties) + self._diagnostics = restore_data_arrays_with_properties( + raw_diagnostics, self.diagnostic_properties) + self._last_update_time = state['time'] + return self._tendencies, self._diagnostics @abc.abstractmethod def array_call(self, state): @@ -249,13 +455,36 @@ class ImplicitPrognostic(object): object is called, and values are dictionaries which indicate 'dims' and 'units'. tendency_properties : dict - A dictionary whose keys are quantities for which tendencies are returned when the - object is called, and values are dictionaries which indicate 'dims' and - 'units'. + A dictionary whose keys are quantities for which tendencies are returned + when the object is called, and values are dictionaries which indicate + 'dims' and 'units'. diagnostic_properties : dict A dictionary whose keys are diagnostic quantities returned when the object is called, and values are dictionaries which indicate 'dims' and 'units'. + input_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + tendency_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which tendency values are scaled before being + returned by this object. + diagnostic_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + tendencies_in_diagnostics : bool + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output. + update_interval : timedelta + If not None, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. + name : string + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". By default the class name in + lowercase is used. """ __metaclass__ = abc.ABCMeta @@ -293,6 +522,59 @@ def __repr__(self): self._making_repr = False return return_value + def __init__(self, + input_scale_factors=None, tendency_scale_factors=None, + diagnostic_scale_factors=None, tendencies_in_diagnostics=False, + update_interval=None, name=None): + """ + Initializes the Implicit object. + + Args + ---- + input_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + tendency_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which tendency values are scaled before being + returned by this object. + diagnostic_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + tendencies_in_diagnostics : bool + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output. + update_interval : timedelta + If given, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. + name : string + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". By default the class name in + lowercase is used. + """ + if input_scale_factors is not None: + self.input_scale_factors = input_scale_factors + else: + self.input_scale_factors = {} + if tendency_scale_factors is not None: + self.tendency_scale_factors = tendency_scale_factors + else: + self.tendency_scale_factors = {} + if diagnostic_scale_factors is not None: + self.diagnostic_scale_factors = diagnostic_scale_factors + else: + self.diagnostic_scale_factors = {} + self.tendencies_in_diagnostics = tendencies_in_diagnostics + self.update_interval = update_interval + self._last_update_time = None + if self.name is None: + self.name = self.__class__.__name__.lower() + else: + self.name = name + def __call__(self, state, timestep): """ Gets tendencies and diagnostics from the passed model state. @@ -323,13 +605,20 @@ def __call__(self, state, timestep): InvalidStateError If state is not a valid input for the Prognostic instance. """ - raw_state = get_numpy_arrays_with_properties(state, self.input_properties) - raw_tendencies, raw_diagnostics = self.array_call(raw_state, timestep) - tendencies = restore_data_arrays_with_properties( - raw_tendencies, self.tendency_properties) - diagnostics = restore_data_arrays_with_properties( - raw_diagnostics, self.diagnostic_properties) - return tendencies, diagnostics + if (self.update_interval is None or + self._last_update_time is None or + state['time'] > self._last_update_time + self.update_interval): + raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + apply_scale_factors(raw_state, self.input_scale_factors) + raw_tendencies, raw_diagnostics = self.array_call(raw_state, timestep) + apply_scale_factors(raw_tendencies, self.tendency_scale_factors) + apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) + self._tendencies = restore_data_arrays_with_properties( + raw_tendencies, self.tendency_properties) + self._diagnostics = restore_data_arrays_with_properties( + raw_diagnostics, self.diagnostic_properties) + self._last_update_time = state['time'] + return self._tendencies, self._diagnostics @abc.abstractmethod def array_call(self, state, timestep): @@ -372,6 +661,18 @@ class Diagnostic(object): A dictionary whose keys are diagnostic quantities returned when the object is called, and values are dictionaries which indicate 'dims' and 'units'. + input_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + diagnostic_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + update_interval : timedelta + If not None, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. """ __metaclass__ = abc.ABCMeta @@ -404,6 +705,39 @@ def __repr__(self): self._making_repr = False return return_value + def __init__(self, + input_scale_factors=None, diagnostic_scale_factors=None, + update_interval=None): + """ + Initializes the Implicit object. + + Args + ---- + input_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + diagnostic_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + update_interval : timedelta + If given, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. + """ + if input_scale_factors is not None: + self.input_scale_factors = input_scale_factors + else: + self.input_scale_factors = {} + if diagnostic_scale_factors is not None: + self.diagnostic_scale_factors = diagnostic_scale_factors + else: + self.diagnostic_scale_factors = {} + self.update_interval = update_interval + self._last_update_time = None + self._diagnostics = None + def __call__(self, state): """ Gets diagnostics from the passed model state. @@ -427,11 +761,16 @@ def __call__(self, state): InvalidStateError If state is not a valid input for the Prognostic instance. """ - raw_state = get_numpy_arrays_with_properties(state, self.input_properties) - raw_diagnostics = self.array_call(raw_state) - diagnostics = restore_data_arrays_with_properties( - raw_diagnostics, self.diagnostic_properties) - return diagnostics + if (self.update_interval is None or + self._last_update_time is None or + state['time'] > self._last_update_time + self.update_interval): + raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + apply_scale_factors(raw_state, self.input_scale_factors) + raw_diagnostics = self.array_call(raw_state) + apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) + self._diagnostics = restore_data_arrays_with_properties( + raw_diagnostics, self.diagnostic_properties) + return self._diagnostics @abc.abstractmethod def array_call(self, state): diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 9d679c7..c43876e 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -1,21 +1,30 @@ from .exceptions import SharedKeyError from .base_components import Prognostic, Diagnostic, Monitor -from sympl._core.util import update_dict_by_adding_another, ensure_no_shared_keys +from .util import update_dict_by_adding_another, ensure_no_shared_keys +import warnings class ComponentComposite(object): + """ + A composite of components that allows them to be called as one object. + + Attributes + ---------- + component_list: list + The components being composited by this object. + """ component_class = None def __str__(self): return '{}(\n{}\n)'.format( self.__class__, - ',\n'.join(str(component) for component in self._components)) + ',\n'.join(str(component) for component in self.component_list)) def __repr__(self): return '{}(\n{}\n)'.format( self.__class__, - ',\n'.join(repr(component) for component in self._components)) + ',\n'.join(repr(component) for component in self.component_list)) def __init__(self, *args): """ @@ -31,17 +40,17 @@ def __init__(self, *args): """ if self.component_class is not None: ensure_components_have_class(args, self.component_class) - self._components = args + self.component_list = args if hasattr(self, 'diagnostics'): if (len(self.diagnostics) != - sum([len(comp.diagnostics) for comp in self._components])): + sum([len(comp.diagnostics) for comp in self.component_list])): raise SharedKeyError( 'Two components in a composite should not compute ' 'the same diagnostic') def _combine_attribute(self, attr): return_attr = [] - for component in self._components: + for component in self.component_list: return_attr.extend(getattr(component, attr)) return tuple(set(return_attr)) # set to deduplicate @@ -61,17 +70,6 @@ def ensure_components_have_class(components, component_class): class PrognosticComposite(ComponentComposite): - """ - Attributes - ---------- - inputs : tuple of str - The quantities required in the state when the object is called. - tendencies : tuple of str - The quantities for which tendencies are returned when - the object is called. - diagnostics : tuple of str - The diagnostic quantities returned when the object is called. - """ component_class = Prognostic @@ -107,34 +105,14 @@ def __call__(self, state): """ return_tendencies = {} return_diagnostics = {} - for prognostic in self._components: + for prognostic in self.component_list: tendencies, diagnostics = prognostic(state) update_dict_by_adding_another(return_tendencies, tendencies) return_diagnostics.update(diagnostics) return return_tendencies, return_diagnostics - @property - def inputs(self): - return self._combine_attribute('inputs') - - @property - def diagnostics(self): - return self._combine_attribute('diagnostics') - - @property - def tendencies(self): - return self._combine_attribute('tendencies') - class DiagnosticComposite(ComponentComposite): - """ - Attributes - ---------- - inputs : tuple of str - The quantities required in the state when the object is called. - diagnostics : tuple of str - The diagnostic quantities returned when the object is called. - """ component_class = Diagnostic @@ -165,21 +143,13 @@ def __call__(self, state): If state is not a valid input for a Diagnostic instance. """ return_diagnostics = {} - for diagnostic_component in self._components: + for diagnostic_component in self.component_list: diagnostics = diagnostic_component(state) # ensure two diagnostics don't compute the same quantity ensure_no_shared_keys(return_diagnostics, diagnostics) return_diagnostics.update(diagnostics) return return_diagnostics - @property - def inputs(self): - return self._combine_attribute('inputs') - - @property - def diagnostics(self): - return self._combine_attribute('diagnostics') - class MonitorComposite(ComponentComposite): @@ -202,5 +172,5 @@ def store(self, state): InvalidStateError If state is not a valid input for a Monitor instance. """ - for monitor in self._components: + for monitor in self.component_list: monitor.store(state) diff --git a/sympl/_core/timestepper.py b/sympl/_core/timestepper.py index ce6bd6c..ae87c6e 100644 --- a/sympl/_core/timestepper.py +++ b/sympl/_core/timestepper.py @@ -1,6 +1,6 @@ import abc - from .composite import PrognosticComposite +from .time import timedelta class TimeStepper(object): @@ -11,10 +11,6 @@ class TimeStepper(object): Attributes ---------- - input_properties : dict - A dictionary whose keys are quantities required in the state when the - object is called, and values are dictionaries which indicate 'dims' and - 'units'. diagnostic_properties : dict A dictionary whose keys are quantities for which values for the old state are returned when the @@ -25,17 +21,34 @@ class TimeStepper(object): for the new state are returned when the object is called, and values are dictionaries which indicate 'dims' and 'units'. + prognostic : PrognosticComposite + A composite of the Prognostic objects used by the TimeStepper + prognostic_list: list of Prognostic + A list of Prognostic objects called by the TimeStepper. These should + be referenced when determining what inputs are necessary for the + TimeStepper. + tendencies_in_diagnostics : bool + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output. + time_unit_name : str + The unit to use for time differencing when putting tendencies in + diagnostics. + time_unit_timedelta: timedelta + A timedelta corresponding to a single time unit as used for time + differencing when putting tendencies in diagnostics. """ - __metaclass__ = abc.ABCMeta - @abc.abstractproperty - def input_properties(self): - return {} + time_unit_name = 's' + time_unit_timedelta = timedelta(seconds=1) - @abc.abstractproperty def diagnostic_properties(self): - return {} + return_value = {} + for prognostic in self.prognostic_list: + return_value.update(prognostic.diagnostics) + if self.tendencies_in_diagnostics: + self._insert_tendencies_to_diagnostic_properties(return_value) + return return_value @abc.abstractproperty def output_properties(self): @@ -44,14 +57,7 @@ def output_properties(self): def __str__(self): return ( 'instance of {}(TimeStepper)\n' - ' inputs: {}\n' - ' outputs: {}\n' - ' diagnostics: {}\n' - ' Prognostic components: {}'.format( - self.__class__, self.input_properties.keys(), - self.output_properties.keys(), - self.diagnostic_properties.keys(), - str(self._prognostic)) + ' Prognostic components: {}'.format(self.prognostic_list) ) def __repr__(self): @@ -67,10 +73,56 @@ def __repr__(self): self._making_repr = False return return_value - def __init__(self, prognostic_list, **kwargs): - self._prognostic = PrognosticComposite(*prognostic_list) + def __init__(self, prognostic_list, tendencies_in_diagnostics=False): + """ + Initialize the TimeStepper. + + Parameters + ---------- + prognostic_list : list of Prognostic + Objects to call for tendencies when doing time stepping. + tendencies_in_diagnostics : bool, optional + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output. + """ + self._tendencies_in_diagnostics = tendencies_in_diagnostics + self.prognostic = PrognosticComposite(prognostic_list) + if tendencies_in_diagnostics: + self._insert_tendencies_to_diagnostic_properties() + + def _insert_tendencies_to_diagnostic_properties( + self, diagnostic_properties): + for quantity_name, properties in self.output_properties.items(): + tendency_name = self._get_tendency_name(quantity_name, component_name) + if properties['units'] is '': + units = '{}^-1'.format(self.time_unit_name) + else: + units = '{} {}^-1'.format( + properties['units'], self.time_unit_name) + diagnostic_properties[tendency_name] = { + 'units': units, + 'dims': properties['dims'], + } + + def _insert_tendencies_to_diagnostics( + self, raw_state, raw_new_state, timestep, raw_diagnostics): + for name in self.output_properties.keys(): + tendency_name = self._get_tendency_name(name) + raw_diagnostics[tendency_name] = ( + (raw_new_state[name] - raw_state[name]) / + timestep.total_seconds() * self.time_unit_timedelta.total_seconds()) + + @property + def prognostic_list(self): + return self.prognostic.component_list + + @property + def tendencies_in_diagnostics(self): # value cannot be modified + return self._tendencies_in_diagnostics + + def _get_tendency_name(self, quantity_name, component_name): + return '{}_tendency_from_{}'.format(quantity_name, component_name) - @abc.abstractmethod def __call__(self, state, timestep): """ Retrieves any diagnostics and returns a new state corresponding @@ -90,3 +142,24 @@ def __call__(self, state, timestep): new_state : dict The model state at the next timestep. """ + + @abc.abstractmethod + def _call(self, state, timestep): + """ + Retrieves any diagnostics and returns a new state corresponding + to the next timestep. + + Args + ---- + state : dict + The current model state. + timestep : timedelta + The amount of time to step forward. + + Returns + ------- + diagnostics : dict + Diagnostics from the timestep of the input state. + new_state : dict + The model state at the next timestep. + """ diff --git a/sympl/_core/wrappers.py b/sympl/_core/wrappers.py index 2190b08..cae0e00 100644 --- a/sympl/_core/wrappers.py +++ b/sympl/_core/wrappers.py @@ -2,274 +2,6 @@ from .array import DataArray -class ScalingWrapper(object): - """ - Wraps any component and scales either inputs, outputs or tendencies - by a floating point value. - Example - ------- - This is how the ScaledInputOutputWrapper can be used to wrap a Prognostic. - >>> scaled_component = ScaledInputOutputWrapper( - >>> RRTMRadiation(), - >>> input_scale_factors = { - >>> 'specific_humidity' = 0.2}, - >>> tendency_scale_factors = { - >>> 'air_temperature' = 1.5}) - """ - - def __init__(self, - component, - input_scale_factors=None, - output_scale_factors=None, - tendency_scale_factors=None, - diagnostic_scale_factors=None): - """ - Initializes the ScaledInputOutputWrapper object. - Args - ---- - component : Prognostic, Implicit - The component to be wrapped. - input_scale_factors : dict - a dictionary whose keys are the inputs that will be scaled - and values are floating point scaling factors. - output_scale_factors : dict - a dictionary whose keys are the outputs that will be scaled - and values are floating point scaling factors. - tendency_scale_factors : dict - a dictionary whose keys are the tendencies that will be scaled - and values are floating point scaling factors. - diagnostic_scale_factors : dict - a dictionary whose keys are the diagnostics that will be scaled - and values are floating point scaling factors. - Returns - ------- - scaled_component : ScaledInputOutputWrapper - the scaled version of the component - Raises - ------ - TypeError - The component is not of type Implicit or Prognostic. - ValueError - The keys in the scale factors do not correspond to valid - input/output/tendency for this component. - """ - - self._input_scale_factors = dict() - if input_scale_factors is not None: - - for input_field in input_scale_factors.keys(): - if input_field not in component.inputs: - raise ValueError( - "{} is not a valid input quantity.".format(input_field)) - - self._input_scale_factors = input_scale_factors - - self._diagnostic_scale_factors = dict() - if diagnostic_scale_factors is not None: - - for diagnostic_field in diagnostic_scale_factors.keys(): - if diagnostic_field not in component.diagnostics: - raise ValueError( - "{} is not a valid diagnostic quantity.".format(diagnostic_field)) - - self._diagnostic_scale_factors = diagnostic_scale_factors - - if hasattr(component, 'input_properties') and hasattr(component, 'output_properties'): - - self._output_scale_factors = dict() - if output_scale_factors is not None: - - for output_field in output_scale_factors.keys(): - if output_field not in component.outputs: - raise ValueError( - "{} is not a valid output quantity.".format(output_field)) - - self._output_scale_factors = output_scale_factors - self._component_type = 'Implicit' - - elif hasattr(component, 'input_properties') and hasattr(component, 'tendency_properties'): - - self._tendency_scale_factors = dict() - if tendency_scale_factors is not None: - - for tendency_field in tendency_scale_factors.keys(): - if tendency_field not in component.tendencies: - raise ValueError( - "{} is not a valid tendency quantity.".format(tendency_field)) - - self._tendency_scale_factors = tendency_scale_factors - self._component_type = 'Prognostic' - - elif hasattr(component, 'input_properties') and hasattr(component, 'diagnostic_properties'): - self._component_type = 'Diagnostic' - else: - raise TypeError( - "Component must be either of type Implicit or Prognostic or Diagnostic") - - self._component = component - - def __getattr__(self, item): - return getattr(self._component, item) - - def __call__(self, state, timestep=None): - - scaled_state = {} - if 'time' in state: - scaled_state['time'] = state['time'] - - for input_field in self.inputs: - if input_field in self._input_scale_factors: - scale_factor = self._input_scale_factors[input_field] - scaled_state[input_field] = state[input_field]*float(scale_factor) - else: - scaled_state[input_field] = state[input_field] - - if self._component_type == 'Implicit': - diagnostics, new_state = self._component(scaled_state, timestep) - - for output_field in self._output_scale_factors.keys(): - scale_factor = self._output_scale_factors[output_field] - new_state[output_field] *= float(scale_factor) - - for diagnostic_field in self._diagnostic_scale_factors.keys(): - scale_factor = self._diagnostic_scale_factors[diagnostic_field] - diagnostics[diagnostic_field] *= float(scale_factor) - - return diagnostics, new_state - elif self._component_type == 'Prognostic': - tendencies, diagnostics = self._component(scaled_state) - - for tend_field in self._tendency_scale_factors.keys(): - scale_factor = self._tendency_scale_factors[tend_field] - tendencies[tend_field] *= float(scale_factor) - - for diagnostic_field in self._diagnostic_scale_factors.keys(): - scale_factor = self._diagnostic_scale_factors[diagnostic_field] - diagnostics[diagnostic_field] *= float(scale_factor) - - return tendencies, diagnostics - elif self._component_type == 'Diagnostic': - diagnostics = self._component(scaled_state) - - for diagnostic_field in self._diagnostic_scale_factors.keys(): - scale_factor = self._diagnostic_scale_factors[diagnostic_field] - diagnostics[diagnostic_field] *= float(scale_factor) - - return diagnostics - else: # Should never reach this - raise ValueError( - 'Unknown component type, seems to be a bug in ScalingWrapper') - - -class UpdateFrequencyWrapper(object): - """ - Wraps a prognostic object so that when it is called, it only computes new - output if sufficient time has passed, and otherwise returns its last - computed output. The Delayed object requires that the 'time' attribute is - set in the state, in addition to any requirements of the Prognostic - - Example - ------- - This how the wrapper should be used on a fictional Prognostic class - called MyPrognostic. - - >>> from datetime import timedelta - >>> prognostic = UpdateFrequencyWrapper(MyPrognostic(), timedelta(hours=1)) - """ - - def __init__(self, prognostic, update_timedelta): - """ - Initialize the UpdateFrequencyWrapper object. - - Args - ---- - prognostic : Prognostic - The object to be wrapped. - update_timedelta : timedelta - The amount that state['time'] must differ from when output - was cached before new output is computed. - """ - self._prognostic = prognostic - self._update_timedelta = update_timedelta - self._cached_output = None - self._last_update_time = None - - def __call__(self, state, **kwargs): - if ((self._last_update_time is None) or - (state['time'] >= self._last_update_time + - self._update_timedelta)): - self._cached_output = self._prognostic(state, **kwargs) - self._last_update_time = state['time'] - return self._cached_output - - def __getattr__(self, item): - return getattr(self._prognostic, item) - - -class TendencyInDiagnosticsWrapper(object): - """ - Wraps a prognostic object so that when it is called, it returns all - tendencies in its diagnostics. - - Example - ------- - This how the wrapper should be used on a fictional Prognostic class - called RRTMRadiation. - - >>> prognostic = TendencyInDiagnosticsWrapper(RRTMRadiation(), 'radiation') - """ - - def __init__(self, prognostic, label): - """ - Initialize the Delayed object. - - Args - ---- - prognostic : Prognostic - The object to be wrapped - label : str - Label describing what the tendencies are due to, to be - put in the diagnostic quantity names. - """ - self._prognostic = prognostic - self._tendency_label = label - self._tendency_diagnostic_properties = {} - for quantity_name, properties in prognostic.tendency_properties.items(): - diagnostic_name = 'tendency_of_{}_due_to_{}'.format(quantity_name, label) - self._tendency_diagnostic_properties[diagnostic_name] = properties - - @property - def inputs(self): - return list(self.diagnostic_properties.keys()) - - @property - def input_properties(self): - return_dict = self._prognostic.input_properties.copy() - return_dict.update(self._tendency_diagnostic_properties) - return return_dict - - @property - def diagnostics(self): - return list(self.diagnostic_properties.keys()) - - @property - def diagnostic_properties(self): - return_dict = self._prognostic.diagnostic_properties.copy() - return_dict.update(self._tendency_diagnostic_properties) - return return_dict - - def __call__(self, state, **kwargs): - tendencies, diagnostics = self._prognostic(state, **kwargs) - for quantity_name in tendencies.keys(): - diagnostic_name = 'tendency_of_{}_due_to_{}'.format( - quantity_name, self._tendency_label) - diagnostics[diagnostic_name] = tendencies[quantity_name] - return tendencies, diagnostics - - def __getattr__(self, item): - return getattr(self._prognostic, item) - - class TimeDifferencingWrapper(ImplicitPrognostic): """ Wraps an Implicit object and turns it into an ImplicitPrognostic by applying From 5ad668d19af4e1be8ec6e04a6defeeeb993379bd Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 19 Mar 2018 16:28:31 -0700 Subject: [PATCH 07/98] added check for netcdftime to have required objects in order to be used --- HISTORY.rst | 5 ++++- sympl/_core/time.py | 4 ++++ tests/test_time.py | 5 ++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 96ad794..f2339e7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,11 +11,14 @@ Latest TendencyInDiagnosticsWrapper is now to be used in Implicit and TimeStepper objects. * Composites now have a component_list attribute which contains the components being composited. +* Added a check for netcdftime having the required objects, to fall back on not + using netcdftime when those are missing. This is because most objects are missing in + older versions of netcdftime (that come packaged with netCDF4). Breaking changes ~~~~~~~~~~~~~~~~ -* inputs, outputs, diagnotsics, and tendencies are no longer attributes of components. +* inputs, outputs, diagnostics, and tendencies are no longer attributes of components. In order to get these, you should use e.g. input_properties.keys() * properties dictionaries are now abstract methods, so subclasses must define them. Previously they defaulted to empty dictionaries. diff --git a/sympl/_core/time.py b/sympl/_core/time.py index 371939a..ee1ff43 100644 --- a/sympl/_core/time.py +++ b/sympl/_core/time.py @@ -2,6 +2,10 @@ from .exceptions import DependencyError try: import netcdftime as nt + if not all(hasattr(nt, attr) for attr in [ + 'DatetimeNoLeap', 'DatetimeProlepticGregorian', 'DatetimeAllLeap', + 'Datetime360Day', 'DatetimeJulian', 'DatetimeGregorian']): + nt = None except ImportError: nt = None diff --git a/tests/test_time.py b/tests/test_time.py index 8a38709..71fcf6a 100644 --- a/tests/test_time.py +++ b/tests/test_time.py @@ -4,6 +4,10 @@ import pytz try: import netcdftime as nt + if not all(hasattr(nt, attr) for attr in [ + 'DatetimeNoLeap', 'DatetimeProlepticGregorian', 'DatetimeAllLeap', + 'Datetime360Day', 'DatetimeJulian', 'DatetimeGregorian']): + nt = None except ImportError: nt = None @@ -48,7 +52,6 @@ def testMaxArgDatetimeHasCorrectValues(self): assert max_args_dt.microsecond == 10 - class ProlepticGregorianTests(unittest.TestCase, DatetimeBase): dt_class = real_datetime From 87d7e54c29537eab514e23760eeef39b53480398 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 21 Mar 2018 10:35:45 -0700 Subject: [PATCH 08/98] Added tests for new base components * Added tests for Implicit, Diagnostic, Prognostic, ImplicitPrognostic * Fixed the code to pass the tests * Moved composite tests to their own file, fixed them to work with the new component style * decided 'time' must be present in model state --- HISTORY.rst | 5 +- sympl/__init__.py | 9 +- sympl/_core/base_components.py | 77 +- tests/test_base_components.py | 2074 ++++++++++++++++++++++--- tests/test_composite.py | 295 ++++ tests/test_get_restore_numpy_array.py | 20 + 6 files changed, 2207 insertions(+), 273 deletions(-) create mode 100644 tests/test_composite.py diff --git a/HISTORY.rst b/HISTORY.rst index f2339e7..0bc1835 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,7 +13,7 @@ Latest composited. * Added a check for netcdftime having the required objects, to fall back on not using netcdftime when those are missing. This is because most objects are missing in - older versions of netcdftime (that come packaged with netCDF4). + older versions of netcdftime (that come packaged with netCDF4) (closes #23). Breaking changes ~~~~~~~~~~~~~~~~ @@ -29,6 +29,9 @@ Breaking changes * ScalingWrapper, UpdateFrequencyWrapper, and TendencyInDiagnosticsWrapper have been removed. The functionality of these wrappers has been moved to the component base types as methods and initialization options. +* 'time' now must be present in the model state dictionary. This is strictly required + for calls to Diagnostic, Prognostic, ImplicitPrognostic, and Implicit components, + and may be strictly required in other ways in the future v0.3.1 ------ diff --git a/sympl/__init__.py b/sympl/__init__.py index 4a5f6a0..21bd4f4 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -19,9 +19,7 @@ restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, set_direction_names, add_direction_names) -from ._core.wrappers import ( - UpdateFrequencyWrapper, TendencyInDiagnosticsWrapper, - TimeDifferencingWrapper, ScalingWrapper) +from ._core.wrappers import TimeDifferencingWrapper from ._core.testing import ComponentTestBase from ._components import ( PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, @@ -37,13 +35,12 @@ InvalidPropertyDictError, DataArray, get_constant, set_constant, set_condensible_name, reset_constants, - UpdateFrequencyWrapper, TimeDifferencingWrapper, combine_dimensions, + TimeDifferencingWrapper, combine_dimensions, ensure_no_shared_keys, - get_numpy_array, jit, TendencyInDiagnosticsWrapper, + get_numpy_array, jit, restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, set_direction_names, add_direction_names, - ScalingWrapper, ComponentTestBase, PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index db4b8b7..b394276 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -2,6 +2,7 @@ from util import ( get_numpy_arrays_with_properties, restore_data_arrays_with_properties) from .time import timedelta +from .exceptions import InvalidPropertyDictError def apply_scale_factors(array_state, scale_factors): @@ -144,14 +145,14 @@ def __init__(self, self._tendencies_in_diagnostics = tendencies_in_diagnostics self.update_interval = update_interval self._last_update_time = None - if self.name is None: + if name is None: self.name = self.__class__.__name__.lower() else: self.name = name if tendencies_in_diagnostics: - self._insert_tendencies_to_diagnostic_properties() + self._insert_tendency_properties() - def _insert_tendencies_to_diagnostic_properties(self): + def _insert_tendency_properties(self): for name, properties in self.output_properties.items(): tendency_name = self._get_tendency_name(name) if properties['units'] is '': @@ -162,6 +163,29 @@ def _insert_tendencies_to_diagnostic_properties(self): 'units': units, 'dims': properties['dims'], } + if name not in self.input_properties.keys(): + self.input_properties[name] = { + 'dims': properties['dims'], + 'units': properties['units'], + } + elif self.input_properties[name]['dims'] != self.output_properties[name]['dims']: + raise InvalidPropertyDictError( + 'Can only calculate tendencies when input and output values' + ' have the same dimensions, but dims for {} are ' + '{} (input) and {} (output)'.format( + name, self.input_properties[name]['dims'], + self.output_properties[name]['dims'] + ) + ) + elif self.input_properties[name]['units'] != self.output_properties[name]['units']: + raise InvalidPropertyDictError( + 'Can only calculate tendencies when input and output values' + ' have the same units, but units for {} are ' + '{} (input) and {} (output)'.format( + name, self.input_properties[name]['units'], + self.output_properties[name]['units'] + ) + ) def _get_tendency_name(self, name): return '{}_tendency_from_{}'.format(name, self.name) @@ -201,19 +225,22 @@ def __call__(self, state, timestep): """ if (self.update_interval is None or self._last_update_time is None or - state['time'] > self._last_update_time + self.update_interval): + state['time'] >= self._last_update_time + self.update_interval): raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + raw_state['time'] = state['time'] apply_scale_factors(raw_state, self.input_scale_factors) - raw_diagnostics, raw_new_state = self.array_call(state, timestep) + raw_diagnostics, raw_new_state = self.array_call(raw_state, timestep) apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) apply_scale_factors(raw_new_state, self.output_scale_factors) if self.tendencies_in_diagnostics: self._insert_tendencies_to_diagnostics( raw_state, raw_new_state, timestep, raw_diagnostics) self._diagnostics = restore_data_arrays_with_properties( - raw_diagnostics, self.diagnostic_properties) + raw_diagnostics, self.diagnostic_properties, + state, self.input_properties) self._new_state = restore_data_arrays_with_properties( - raw_new_state, self.output_properties) + raw_new_state, self.output_properties, + state, self.input_properties) self._last_update_time = state['time'] return self._diagnostics, self._new_state @@ -370,7 +397,7 @@ def __init__(self, self.diagnostic_scale_factors = {} self.update_interval = update_interval self._last_update_time = None - if self.name is None: + if name is None: self.name = self.__class__.__name__.lower() else: self.name = name @@ -405,16 +432,19 @@ def __call__(self, state): """ if (self.update_interval is None or self._last_update_time is None or - state['time'] > self._last_update_time + self.update_interval): + state['time'] >= self._last_update_time + self.update_interval): raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + raw_state['time'] = state['time'] apply_scale_factors(raw_state, self.input_scale_factors) raw_tendencies, raw_diagnostics = self.array_call(raw_state) apply_scale_factors(raw_tendencies, self.tendency_scale_factors) apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) self._tendencies = restore_data_arrays_with_properties( - raw_tendencies, self.tendency_properties) + raw_tendencies, self.tendency_properties, + state, self.input_properties) self._diagnostics = restore_data_arrays_with_properties( - raw_diagnostics, self.diagnostic_properties) + raw_diagnostics, self.diagnostic_properties, + state, self.input_properties) self._last_update_time = state['time'] return self._tendencies, self._diagnostics @@ -474,9 +504,6 @@ class ImplicitPrognostic(object): A (possibly empty) dictionary whose keys are quantity names and values are floats by which diagnostic values are scaled before being returned by this object. - tendencies_in_diagnostics : bool - A boolean indicating whether this object will put tendencies of - quantities in its diagnostic output. update_interval : timedelta If not None, the component will only give new output if at least a period of update_interval has passed since the last time new @@ -524,8 +551,7 @@ def __repr__(self): def __init__(self, input_scale_factors=None, tendency_scale_factors=None, - diagnostic_scale_factors=None, tendencies_in_diagnostics=False, - update_interval=None, name=None): + diagnostic_scale_factors=None, update_interval=None, name=None): """ Initializes the Implicit object. @@ -567,10 +593,9 @@ def __init__(self, self.diagnostic_scale_factors = diagnostic_scale_factors else: self.diagnostic_scale_factors = {} - self.tendencies_in_diagnostics = tendencies_in_diagnostics self.update_interval = update_interval self._last_update_time = None - if self.name is None: + if name is None: self.name = self.__class__.__name__.lower() else: self.name = name @@ -607,16 +632,19 @@ def __call__(self, state, timestep): """ if (self.update_interval is None or self._last_update_time is None or - state['time'] > self._last_update_time + self.update_interval): + state['time'] >= self._last_update_time + self.update_interval): raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + raw_state['time'] = state['time'] apply_scale_factors(raw_state, self.input_scale_factors) raw_tendencies, raw_diagnostics = self.array_call(raw_state, timestep) apply_scale_factors(raw_tendencies, self.tendency_scale_factors) apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) self._tendencies = restore_data_arrays_with_properties( - raw_tendencies, self.tendency_properties) + raw_tendencies, self.tendency_properties, + state, self.input_properties) self._diagnostics = restore_data_arrays_with_properties( - raw_diagnostics, self.diagnostic_properties) + raw_diagnostics, self.diagnostic_properties, + state, self.input_properties) self._last_update_time = state['time'] return self._tendencies, self._diagnostics @@ -763,13 +791,16 @@ def __call__(self, state): """ if (self.update_interval is None or self._last_update_time is None or - state['time'] > self._last_update_time + self.update_interval): + state['time'] >= self._last_update_time + self.update_interval): raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + raw_state['time'] = state['time'] apply_scale_factors(raw_state, self.input_scale_factors) raw_diagnostics = self.array_call(raw_state) apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) self._diagnostics = restore_data_arrays_with_properties( - raw_diagnostics, self.diagnostic_properties) + raw_diagnostics, self.diagnostic_properties, + state, self.input_properties) + self._last_update_time = state['time'] return self._diagnostics @abc.abstractmethod diff --git a/tests/test_base_components.py b/tests/test_base_components.py index f030814..0e3e3f9 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -1,8 +1,10 @@ import pytest import mock +import numpy as np +import unittest from sympl import ( - Prognostic, Diagnostic, Monitor, PrognosticComposite, DiagnosticComposite, - MonitorComposite, SharedKeyError, DataArray + Prognostic, Diagnostic, Monitor, Implicit, ImplicitPrognostic, + datetime, timedelta, DataArray, InvalidPropertyDictError ) def same_list(list1, list2): @@ -12,20 +14,100 @@ def same_list(list1, list2): class MockPrognostic(Prognostic): - def __call__(self, state): - return {}, {} + input_properties = None + diagnostic_properties = None + tendency_properties = None + def __init__( + self, input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, **kwargs): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self.tendency_properties = tendency_properties + self._diagnostic_output = diagnostic_output + self._tendency_output = tendency_output + self.times_called = 0 + self.state_given = None + super(MockPrognostic, self).__init__(**kwargs) -class MockPrognostic2(Prognostic): + def array_call(self, state): + self.times_called += 1 + self.state_given = state + return self._tendency_output, self._diagnostic_output - def __call__(self, state): - return {}, {} + +class MockImplicitPrognostic(ImplicitPrognostic): + + input_properties = None + diagnostic_properties = None + tendency_properties = None + + def __init__( + self, input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, **kwargs): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self.tendency_properties = tendency_properties + self._diagnostic_output = diagnostic_output + self._tendency_output = tendency_output + self.times_called = 0 + self.state_given = None + self.timestep_given = None + super(MockImplicitPrognostic, self).__init__(**kwargs) + + def array_call(self, state, timestep): + self.times_called += 1 + self.state_given = state + self.timestep_given = timestep + return self._tendency_output, self._diagnostic_output class MockDiagnostic(Diagnostic): - def __call__(self, state): - return {} + input_properties = None + diagnostic_properties = None + + def __init__( + self, input_properties, diagnostic_properties, diagnostic_output, + **kwargs): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self._diagnostic_output = diagnostic_output + self.times_called = 0 + self.state_given = None + super(MockDiagnostic, self).__init__(**kwargs) + + def array_call(self, state): + self.times_called += 1 + self.state_given = state + return self._diagnostic_output + + +class MockImplicit(Implicit): + + input_properties = None + diagnostic_properties = None + output_properties = None + + def __init__( + self, input_properties, diagnostic_properties, output_properties, + diagnostic_output, state_output, + **kwargs): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self.output_properties = output_properties + self._diagnostic_output = diagnostic_output + self._state_output = state_output + self.times_called = 0 + self.state_given = None + self.timestep_given = None + super(MockImplicit, self).__init__(**kwargs) + + def array_call(self, state, timestep): + self.times_called += 1 + self.state_given = state + self.timestep_given = timestep + return self._diagnostic_output, self._state_output class MockMonitor(Monitor): @@ -34,240 +116,1746 @@ def store(self, state): return -def test_empty_prognostic_composite(): - prognostic_composite = PrognosticComposite() - state = {'air_temperature': 273.15} - tendencies, diagnostics = prognostic_composite(state) - assert len(tendencies) == 0 - assert len(diagnostics) == 0 - assert isinstance(tendencies, dict) - assert isinstance(diagnostics, dict) - - -@mock.patch.object(MockPrognostic, '__call__') -def test_prognostic_composite_calls_one_prognostic(mock_call): - mock_call.return_value = ({'air_temperature': 0.5}, {'foo': 50.}) - prognostic_composite = PrognosticComposite(MockPrognostic()) - state = {'air_temperature': 273.15} - tendencies, diagnostics = prognostic_composite(state) - assert mock_call.called - assert tendencies == {'air_temperature': 0.5} - assert diagnostics == {'foo': 50.} - - -@mock.patch.object(MockPrognostic, '__call__') -def test_prognostic_composite_calls_two_prognostics(mock_call): - mock_call.return_value = ({'air_temperature': 0.5}, {}) - prognostic_composite = PrognosticComposite( - MockPrognostic(), MockPrognostic()) - state = {'air_temperature': 273.15} - tendencies, diagnostics = prognostic_composite(state) - assert mock_call.called - assert mock_call.call_count == 2 - assert tendencies == {'air_temperature': 1.} - assert diagnostics == {} - - -def test_empty_diagnostic_composite(): - diagnostic_composite = DiagnosticComposite() - state = {'air_temperature': 273.15} - diagnostics = diagnostic_composite(state) - assert len(diagnostics) == 0 - assert isinstance(diagnostics, dict) - - -@mock.patch.object(MockDiagnostic, '__call__') -def test_diagnostic_composite_calls_one_diagnostic(mock_call): - mock_call.return_value = {'foo': 50.} - diagnostic_composite = DiagnosticComposite(MockDiagnostic()) - state = {'air_temperature': 273.15} - diagnostics = diagnostic_composite(state) - assert mock_call.called - assert diagnostics == {'foo': 50.} - - -def test_empty_monitor_collection(): - # mainly we're testing that nothing errors - monitor_collection = MonitorComposite() - state = {'air_temperature': 273.15} - monitor_collection.store(state) - - -@mock.patch.object(MockMonitor, 'store') -def test_monitor_collection_calls_one_monitor(mock_store): - mock_store.return_value = None - monitor_collection = MonitorComposite(MockMonitor()) - state = {'air_temperature': 273.15} - monitor_collection.store(state) - assert mock_store.called - - -@mock.patch.object(MockMonitor, 'store') -def test_monitor_collection_calls_two_monitors(mock_store): - mock_store.return_value = None - monitor_collection = MonitorComposite(MockMonitor(), MockMonitor()) - state = {'air_temperature': 273.15} - monitor_collection.store(state) - assert mock_store.called - assert mock_store.call_count == 2 - - -def test_prognostic_composite_cannot_use_diagnostic(): - try: - PrognosticComposite(MockDiagnostic()) - except TypeError: - pass - except Exception as err: - raise err - else: - raise AssertionError('TypeError should have been raised') - - -def test_diagnostic_composite_cannot_use_prognostic(): - try: - DiagnosticComposite(MockPrognostic()) - except TypeError: - pass - except Exception as err: - raise err - else: - raise AssertionError('TypeError should have been raised') - - -@mock.patch.object(MockDiagnostic, '__call__') -def test_diagnostic_composite_call(mock_call): - mock_call.return_value = {'foo': 5.} - state = {'bar': 10.} - diagnostics = DiagnosticComposite(MockDiagnostic()) - new_state = diagnostics(state) - assert list(state.keys()) == ['bar'] - assert state['bar'] == 10. - assert list(new_state.keys()) == ['foo'] - assert new_state['foo'] == 5. - - -def test_prognostic_composite_includes_attributes(): - prognostic = MockPrognostic() - prognostic.input_properties = {'input1': {'units': 'K', 'dims':['z', 'y', 'x']}} - prognostic.diagnostic_properties = {'diagnostic1': {'units': 'm/s'}} - prognostic.tendency_properties = {'tendency1': {}} - composite = PrognosticComposite(prognostic) - assert same_list(composite.input_properties.keys(), ('input1',)) - assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1',)) - assert same_list(composite.tendency_properties.keys(), ('tendency1',)) - - -def test_prognostic_composite_includes_attributes_from_two(): - prognostic1 = MockPrognostic() - prognostic1.input_properties = {'input1': {'units': 'K', 'dims':['z', 'y', 'x']}} - prognostic1.diagnostic_properties = {'diagnostic1': {'units': 'm/s'}} - prognostic1.tendency_properties = {'tendency1': {}} - prognostic2 = MockPrognostic() - prognostic2.input_properties = {'input2': {'units': 'K', 'dims':['z', 'y', 'x']}} - prognostic2.diagnostic_properties = {'diagnostic2': {'units': 'm/s'}} - prognostic2.tendency_properties = {'tendency2': {}} - composite = PrognosticComposite(prognostic1, prognostic2) - assert same_list(composite.input_properties.keys(), ('input1', 'input2')) - assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) - assert same_list(composite.tendency_properties.keys(), ('tendency1', 'tendency2')) - - -def test_prognostic_merges_attributes(): - prognostic1 = MockPrognostic() - prognostic1.input_properties = {'input1': {}} - prognostic1.diagnostic_properties = {'diagnostic1': {}} - prognostic1.tendency_properties = {'tendency1': {}, 'tendency2': {}} - prognostic2 = MockPrognostic() - prognostic2.input_properties = {'input1': {}, 'input2': {}} - prognostic2.diagnostic_properties = {'diagnostic2': {}} - prognostic2.tendency_properties = {'tendency2': {}} - composite = PrognosticComposite(prognostic1, prognostic2) - assert same_list(composite.input_properties.keys(), ('input1', 'input2')) - assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) - assert same_list(composite.tendency_properties.keys(), ('tendency1', 'tendency2')) - - -def test_prognostic_composite_ensures_valid_state(): - prognostic1 = MockPrognostic() - prognostic1.input_properties = {'input1': {}} - prognostic1.diagnostic_properties = {'diagnostic1': {}} - prognostic2 = MockPrognostic() - prognostic2.input_properties = {'input1': {}, 'input2': {}} - prognostic2.diagnostic_properties = {'diagnostic1': {}} - try: - PrognosticComposite(prognostic1, prognostic2) - except SharedKeyError: - pass - except Exception as err: - raise err - else: - raise AssertionError( - 'Should not be able to have overlapping diagnostics in composite') - - -@mock.patch.object(MockPrognostic, '__call__') -@mock.patch.object(MockPrognostic2, '__call__') -def test_prognostic_component_handles_units_when_combining(mock_call, mock2_call): - mock_call.return_value = ({ - 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})}, {}) - mock2_call.return_value = ({ - 'eastward_wind': DataArray(50., attrs={'units': 'cm/s'})}, {}) - prognostic1 = MockPrognostic() - prognostic2 = MockPrognostic2() - composite = PrognosticComposite(prognostic1, prognostic2) - tendencies, diagnostics = composite({}) - assert tendencies['eastward_wind'].to_units('m/s').values.item() == 1.5 - - -def test_diagnostic_composite_includes_attributes(): - diagnostic = MockDiagnostic() - diagnostic.input_properties = {'input1': {}} - diagnostic.diagnostic_properties = {'diagnostic1': {}} - composite = DiagnosticComposite(diagnostic) - assert same_list(composite.input_properties.keys(), ('input1',)) - assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1',)) - - -def test_diagnostic_composite_includes_attributes_from_two(): - diagnostic1 = MockDiagnostic() - diagnostic1.input_properties = {'input1': {}} - diagnostic1.diagnostic_properties = {'diagnostic1': {}} - diagnostic2 = MockDiagnostic() - diagnostic2.input_properties = {'input2': {}} - diagnostic2.diagnostic_properties = {'diagnostic2': {}} - composite = DiagnosticComposite(diagnostic1, diagnostic2) - assert same_list(composite.input_properties.keys(), ('input1', 'input2')) - assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) - - -def test_diagnostic_composite_merges_attributes(): - diagnostic1 = MockDiagnostic() - diagnostic1.input_properties = {'input1': {}} - diagnostic1.diagnostic_properties = {'diagnostic1': {}} - diagnostic2 = MockDiagnostic() - diagnostic2.input_properties = {'input1': {}, 'input2': {}} - diagnostic2.diagnostic_properties = {'diagnostic2': {}} - composite = DiagnosticComposite(diagnostic1, diagnostic2) - assert same_list(composite.input_properties.keys(), ('input1', 'input2')) - assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) - - -def test_diagnostic_composite_ensures_valid_state(): - diagnostic1 = MockDiagnostic() - diagnostic1.input_properties = {'input1': {}} - diagnostic1.diagnostic_properties = {'diagnostic1': {}} - diagnostic2 = MockDiagnostic() - diagnostic2.input_properties = {'input1': {}, 'input2': {}} - diagnostic2.diagnostic_properties = {'diagnostic1': {}} - try: - DiagnosticComposite(diagnostic1, diagnostic2) - except SharedKeyError: - pass - except Exception as err: - raise err - else: - raise AssertionError( - 'Should not be able to have overlapping diagnostics in composite') +class PrognosticTests(unittest.TestCase): + + component_class = MockPrognostic + + def call_component(self, component, state): + return component(state) + + def test_empty_prognostic(self): + prognostic = self.component_class({}, {}, {}, {}, {}) + tendencies, diagnostics = self.call_component( + prognostic, {'time': timedelta(seconds=0)}) + assert tendencies == {} + assert diagnostics == {} + assert len(prognostic.state_given) == 1 + assert 'time' in prognostic.state_given.keys() + assert prognostic.state_given['time'] == timedelta(seconds=0) + assert prognostic.times_called == 1 + + def test_input_no_transformations(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _, _ = self.call_component(prognostic, state) + assert len(prognostic.state_given) == 2 + assert 'time' in prognostic.state_given.keys() + assert 'input1' in prognostic.state_given.keys() + assert isinstance(prognostic.state_given['input1'], np.ndarray) + assert np.all(prognostic.state_given['input1'] == np.ones([10])) + + def test_input_converts_units(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'km'} + ) + } + _, _ = self.call_component(prognostic, state) + assert len(prognostic.state_given) == 2 + assert 'time' in prognostic.state_given.keys() + assert 'input1' in prognostic.state_given.keys() + assert isinstance(prognostic.state_given['input1'], np.ndarray) + assert np.all(prognostic.state_given['input1'] == np.ones([10])*1000.) + + def test_input_collects_one_dimension(self): + input_properties = { + 'input1': { + 'dims': ['*'], + 'units': 'm' + } + } + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _, _ = self.call_component(prognostic, state) + assert len(prognostic.state_given) == 2 + assert 'time' in prognostic.state_given.keys() + assert 'input1' in prognostic.state_given.keys() + assert isinstance(prognostic.state_given['input1'], np.ndarray) + assert np.all(prognostic.state_given['input1'] == np.ones([10])) + + def test_input_is_aliased(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'in1', + } + } + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _, _ = self.call_component(prognostic, state) + assert len(prognostic.state_given) == 2 + assert 'time' in prognostic.state_given.keys() + assert 'in1' in prognostic.state_given.keys() + assert isinstance(prognostic.state_given['in1'], np.ndarray) + assert np.all(prognostic.state_given['in1'] == np.ones([10])) + + def test_tendencies_no_transformations(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + }} + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]), + } + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = {'time': timedelta(0)} + tendencies, _ = self.call_component(prognostic, state) + assert len(tendencies) == 1 + assert 'output1' in tendencies.keys() + assert isinstance(tendencies['output1'], DataArray) + assert len(tendencies['output1'].dims) == 1 + assert 'dim1' in tendencies['output1'].dims + assert 'units' in tendencies['output1'].attrs + assert tendencies['output1'].attrs['units'] == 'm/s' + assert np.all(tendencies['output1'].values == np.ones([10])) + + def test_tendencies_with_alias(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s', + 'alias': 'out1', + }} + diagnostic_output = {} + tendency_output = { + 'out1': np.ones([10]), + } + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = {'time': timedelta(0)} + tendencies, _ = self.call_component(prognostic, state) + assert len(tendencies) == 1 + assert 'output1' in tendencies.keys() + assert isinstance(tendencies['output1'], DataArray) + assert len(tendencies['output1'].dims) == 1 + assert 'dim1' in tendencies['output1'].dims + assert 'units' in tendencies['output1'].attrs + assert tendencies['output1'].attrs['units'] == 'm/s' + assert np.all(tendencies['output1'].values == np.ones([10])) + + def test_tendencies_with_alias_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'out1', + } + } + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s', + } + } + diagnostic_output = {} + tendency_output = { + 'out1': np.ones([10]), + } + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + tendencies, _ = self.call_component(prognostic, state) + assert len(tendencies) == 1 + assert 'output1' in tendencies.keys() + assert isinstance(tendencies['output1'], DataArray) + assert len(tendencies['output1'].dims) == 1 + assert 'dim1' in tendencies['output1'].dims + assert 'units' in tendencies['output1'].attrs + assert tendencies['output1'].attrs['units'] == 'm/s' + assert np.all(tendencies['output1'].values == np.ones([10])) + + def test_tendencies_with_dims_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'units': 'm/s', + } + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]), + } + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + tendencies, _ = self.call_component(prognostic, state) + assert len(tendencies) == 1 + assert 'output1' in tendencies.keys() + assert isinstance(tendencies['output1'], DataArray) + assert len(tendencies['output1'].dims) == 1 + assert 'dim1' in tendencies['output1'].dims + assert 'units' in tendencies['output1'].attrs + assert tendencies['output1'].attrs['units'] == 'm/s' + assert np.all(tendencies['output1'].values == np.ones([10])) + + def test_diagnostics_no_transformations(self): + input_properties = {} + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + tendency_properties = {} + diagnostic_output = { + 'output1': np.ones([10]), + } + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = {'time': timedelta(0)} + _, diagnostics = self.call_component(prognostic, state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_with_alias(self): + input_properties = {} + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'out1', + } + } + tendency_properties = {} + diagnostic_output = { + 'out1': np.ones([10]), + } + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = {'time': timedelta(0)} + _, diagnostics = self.call_component(prognostic, state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_with_alias_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'out1', + } + } + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + tendency_properties = {} + diagnostic_output = { + 'out1': np.ones([10]), + } + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _, diagnostics = self.call_component(prognostic, state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_with_dims_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + diagnostic_properties = { + 'output1': { + 'units': 'm', + } + } + tendency_properties = {} + diagnostic_output = { + 'output1': np.ones([10]), + } + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _, diagnostics = self.call_component(prognostic, state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_input_scaling(self): + input_scale_factors = {'input1': 2.} + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, + input_scale_factors=input_scale_factors + ) + assert prognostic.tendency_scale_factors == {} + assert prognostic.diagnostic_scale_factors == {} + assert len(prognostic.input_scale_factors) == 1 + assert 'input1' in prognostic.input_scale_factors.keys() + assert prognostic.input_scale_factors['input1'] == 2. + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _, _ = self.call_component(prognostic, state) + assert len(prognostic.state_given) == 2 + assert 'time' in prognostic.state_given.keys() + assert 'input1' in prognostic.state_given.keys() + assert isinstance(prognostic.state_given['input1'], np.ndarray) + assert np.all(prognostic.state_given['input1'] == np.ones([10])*2.) + + def test_tendency_scaling(self): + tendency_scale_factors = {'output1': 3.} + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + } + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]), + } + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, + tendency_scale_factors=tendency_scale_factors, + ) + assert prognostic.input_scale_factors == {} + assert prognostic.diagnostic_scale_factors == {} + assert len(prognostic.tendency_scale_factors) == 1 + assert 'output1' in prognostic.tendency_scale_factors.keys() + assert prognostic.tendency_scale_factors['output1'] == 3. + state = {'time': timedelta(0)} + tendencies, _ = self.call_component(prognostic, state) + assert len(tendencies) == 1 + assert 'output1' in tendencies.keys() + assert isinstance(tendencies['output1'], DataArray) + assert len(tendencies['output1'].dims) == 1 + assert 'dim1' in tendencies['output1'].dims + assert 'units' in tendencies['output1'].attrs + assert tendencies['output1'].attrs['units'] == 'm/s' + assert np.all(tendencies['output1'].values == np.ones([10])*3.) + + def test_diagnostics_scaling(self): + diagnostic_scale_factors = {'output1': 0.} + input_properties = {} + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + tendency_properties = {} + diagnostic_output = { + 'output1': np.ones([10]), + } + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, + diagnostic_scale_factors=diagnostic_scale_factors, + ) + assert prognostic.tendency_scale_factors == {} + assert prognostic.input_scale_factors == {} + assert len(prognostic.diagnostic_scale_factors) == 1 + assert 'output1' in prognostic.diagnostic_scale_factors.keys() + assert prognostic.diagnostic_scale_factors['output1'] == 0. + state = {'time': timedelta(0)} + _, diagnostics = self.call_component(prognostic, state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])*0.) + + def test_update_interval_on_timedelta(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, update_interval=timedelta(seconds=10) + ) + _, _ = self.call_component(prognostic, {'time': timedelta(seconds=0)}) + assert prognostic.times_called == 1 + _, _ = self.call_component(prognostic, {'time': timedelta(seconds=0)}) + assert prognostic.times_called == 1, 'should not re-compute output' + _, _ = self.call_component(prognostic, {'time': timedelta(seconds=5)}) + assert prognostic.times_called == 1, 'should not re-compute output' + _, _ = self.call_component(prognostic, {'time': timedelta(seconds=10)}) + assert prognostic.times_called == 2, 'should re-compute output' + _, _ = self.call_component(prognostic, {'time': timedelta(seconds=15)}) + assert prognostic.times_called == 2, 'should not re-compute output' + _, _ = self.call_component(prognostic, {'time': timedelta(seconds=20)}) + assert prognostic.times_called == 3, 'should re-compute output' + _, _ = self.call_component(prognostic, {'time': timedelta(seconds=30)}) + assert prognostic.times_called == 4, 'should re-compute output' + _, _ = self.call_component(prognostic, {'time': timedelta(seconds=45)}) + assert prognostic.times_called == 5, 'should re-compute output' + _, _ = self.call_component(prognostic, {'time': timedelta(seconds=50)}) + assert prognostic.times_called == 5, 'should not re-compute output' + + def test_update_interval_on_datetime(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, + update_interval=timedelta(seconds=10) + ) + dt = datetime(2010, 1, 1) + _, _ = self.call_component( + prognostic, {'time': dt + timedelta(seconds=0)}) + assert prognostic.times_called == 1 + _, _ = self.call_component( + prognostic, {'time': dt + timedelta(seconds=0)}) + assert prognostic.times_called == 1, 'should not re-compute output' + _, _ = self.call_component( + prognostic, {'time': dt + timedelta(seconds=5)}) + assert prognostic.times_called == 1, 'should not re-compute output' + _, _ = self.call_component( + prognostic, {'time': dt + timedelta(seconds=10)}) + assert prognostic.times_called == 2, 'should re-compute output' + _, _ = self.call_component( + prognostic, {'time': dt + timedelta(seconds=15)}) + assert prognostic.times_called == 2, 'should not re-compute output' + _, _ = self.call_component( + prognostic, {'time': dt + timedelta(seconds=20)}) + assert prognostic.times_called == 3, 'should re-compute output' + _, _ = self.call_component( + prognostic, {'time': dt + timedelta(seconds=30)}) + assert prognostic.times_called == 4, 'should re-compute output' + _, _ = self.call_component( + prognostic, {'time': dt + timedelta(seconds=45)}) + assert prognostic.times_called == 5, 'should re-compute output' + _, _ = self.call_component( + prognostic, {'time': dt + timedelta(seconds=50)}) + assert prognostic.times_called == 5, 'should not re-compute output' + + +class ImplicitPrognosticTests(PrognosticTests): + + component_class = MockImplicitPrognostic + + def call_component(self, component, state): + return component(state, timedelta(seconds=1)) + + def test_timedelta_is_passed(self): + prognostic = MockImplicitPrognostic({}, {}, {}, {}, {}) + tendencies, diagnostics = prognostic( + {'time': timedelta(seconds=0)}, timedelta(seconds=5)) + assert tendencies == {} + assert diagnostics == {} + assert prognostic.timestep_given == timedelta(seconds=5) + assert prognostic.times_called == 1 + + +class DiagnosticTests(unittest.TestCase): + + component_class = MockDiagnostic + + def call_component(self, component, state): + return component(state) + + def test_empty_diagnostic(self): + diagnostic = self.component_class({}, {}, {}) + diagnostics = diagnostic({'time': timedelta(seconds=0)}) + assert diagnostics == {} + assert len(diagnostic.state_given) == 1 + assert 'time' in diagnostic.state_given.keys() + assert diagnostic.state_given['time'] == timedelta(seconds=0) + assert diagnostic.times_called == 1 + + def test_input_no_transformations(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_properties = {} + diagnostic_output = {} + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, diagnostic_output + ) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _ = diagnostic(state) + assert len(diagnostic.state_given) == 2 + assert 'time' in diagnostic.state_given.keys() + assert 'input1' in diagnostic.state_given.keys() + assert isinstance(diagnostic.state_given['input1'], np.ndarray) + assert np.all(diagnostic.state_given['input1'] == np.ones([10])) + + def test_input_converts_units(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_properties = {} + diagnostic_output = {} + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, + diagnostic_output + ) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'km'} + ) + } + _ = diagnostic(state) + assert len(diagnostic.state_given) == 2 + assert 'time' in diagnostic.state_given.keys() + assert 'input1' in diagnostic.state_given.keys() + assert isinstance(diagnostic.state_given['input1'], np.ndarray) + assert np.all(diagnostic.state_given['input1'] == np.ones([10])*1000.) + + def test_input_collects_one_dimension(self): + input_properties = { + 'input1': { + 'dims': ['*'], + 'units': 'm' + } + } + diagnostic_properties = {} + diagnostic_output = {} + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, + diagnostic_output + ) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _ = diagnostic(state) + assert len(diagnostic.state_given) == 2 + assert 'time' in diagnostic.state_given.keys() + assert 'input1' in diagnostic.state_given.keys() + assert isinstance(diagnostic.state_given['input1'], np.ndarray) + assert np.all(diagnostic.state_given['input1'] == np.ones([10])) + + def test_input_is_aliased(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'in1', + } + } + diagnostic_properties = {} + diagnostic_output = {} + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, + diagnostic_output + ) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _ = diagnostic(state) + assert len(diagnostic.state_given) == 2 + assert 'time' in diagnostic.state_given.keys() + assert 'in1' in diagnostic.state_given.keys() + assert isinstance(diagnostic.state_given['in1'], np.ndarray) + assert np.all(diagnostic.state_given['in1'] == np.ones([10])) + + def test_diagnostics_no_transformations(self): + input_properties = {} + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_output = { + 'output1': np.ones([10]), + } + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, + diagnostic_output + ) + state = {'time': timedelta(0)} + diagnostics = diagnostic(state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_with_alias(self): + input_properties = {} + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'out1', + } + } + diagnostic_output = { + 'out1': np.ones([10]), + } + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, + diagnostic_output + ) + state = {'time': timedelta(0)} + diagnostics = diagnostic(state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_with_alias_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'out1', + } + } + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + diagnostic_output = { + 'out1': np.ones([10]), + } + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, + diagnostic_output + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + diagnostics = diagnostic(state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_with_dims_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + diagnostic_properties = { + 'output1': { + 'units': 'm', + } + } + diagnostic_output = { + 'output1': np.ones([10]), + } + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, + diagnostic_output + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + diagnostics = diagnostic(state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_input_scaling(self): + input_scale_factors = {'input1': 2.} + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_properties = {} + diagnostic_output = {} + diagnostic = self.component_class( + input_properties, diagnostic_properties, + diagnostic_output, + input_scale_factors=input_scale_factors + ) + assert diagnostic.diagnostic_scale_factors == {} + assert len(diagnostic.input_scale_factors) == 1 + assert 'input1' in diagnostic.input_scale_factors.keys() + assert diagnostic.input_scale_factors['input1'] == 2. + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _ = self.call_component(diagnostic, state) + assert len(diagnostic.state_given) == 2 + assert 'time' in diagnostic.state_given.keys() + assert 'input1' in diagnostic.state_given.keys() + assert isinstance(diagnostic.state_given['input1'], np.ndarray) + assert np.all(diagnostic.state_given['input1'] == np.ones([10]) * 2.) + + def test_diagnostics_scaling(self): + diagnostic_scale_factors = {'output1': 0.} + input_properties = {} + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_output = { + 'output1': np.ones([10]), + } + diagnostic = self.component_class( + input_properties, diagnostic_properties, + diagnostic_output, + diagnostic_scale_factors=diagnostic_scale_factors, + ) + assert diagnostic.input_scale_factors == {} + assert len(diagnostic.diagnostic_scale_factors) == 1 + assert 'output1' in diagnostic.diagnostic_scale_factors.keys() + assert diagnostic.diagnostic_scale_factors['output1'] == 0. + state = {'time': timedelta(0)} + diagnostics = self.call_component(diagnostic, state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10]) * 0.) + + def test_update_interval_on_timedelta(self): + input_properties = {} + diagnostic_properties = {} + diagnostic_output = {} + diagnostic = self.component_class( + input_properties, diagnostic_properties, + diagnostic_output, + update_interval=timedelta(seconds=10) + ) + _ = self.call_component(diagnostic, {'time': timedelta(seconds=0)}) + assert diagnostic.times_called == 1 + _ = self.call_component(diagnostic, {'time': timedelta(seconds=0)}) + assert diagnostic.times_called == 1, 'should not re-compute output' + _ = self.call_component(diagnostic, {'time': timedelta(seconds=5)}) + assert diagnostic.times_called == 1, 'should not re-compute output' + _ = self.call_component(diagnostic, {'time': timedelta(seconds=10)}) + assert diagnostic.times_called == 2, 'should re-compute output' + _ = self.call_component(diagnostic, {'time': timedelta(seconds=15)}) + assert diagnostic.times_called == 2, 'should not re-compute output' + _ = self.call_component(diagnostic, {'time': timedelta(seconds=20)}) + assert diagnostic.times_called == 3, 'should re-compute output' + _ = self.call_component(diagnostic, {'time': timedelta(seconds=30)}) + assert diagnostic.times_called == 4, 'should re-compute output' + _ = self.call_component(diagnostic, {'time': timedelta(seconds=45)}) + assert diagnostic.times_called == 5, 'should re-compute output' + _ = self.call_component(diagnostic, {'time': timedelta(seconds=50)}) + assert diagnostic.times_called == 5, 'should not re-compute output' + + def test_update_interval_on_datetime(self): + input_properties = {} + diagnostic_properties = {} + diagnostic_output = {} + diagnostic = self.component_class( + input_properties, diagnostic_properties, + diagnostic_output, + update_interval=timedelta(seconds=10) + ) + dt = datetime(2010, 1, 1) + _ = self.call_component( + diagnostic, {'time': dt + timedelta(seconds=0)}) + assert diagnostic.times_called == 1 + _ = self.call_component( + diagnostic, {'time': dt + timedelta(seconds=0)}) + assert diagnostic.times_called == 1, 'should not re-compute output' + _ = self.call_component( + diagnostic, {'time': dt + timedelta(seconds=5)}) + assert diagnostic.times_called == 1, 'should not re-compute output' + _ = self.call_component( + diagnostic, {'time': dt + timedelta(seconds=10)}) + assert diagnostic.times_called == 2, 'should re-compute output' + _ = self.call_component( + diagnostic, {'time': dt + timedelta(seconds=15)}) + assert diagnostic.times_called == 2, 'should not re-compute output' + _ = self.call_component( + diagnostic, {'time': dt + timedelta(seconds=20)}) + assert diagnostic.times_called == 3, 'should re-compute output' + _ = self.call_component( + diagnostic, {'time': dt + timedelta(seconds=30)}) + assert diagnostic.times_called == 4, 'should re-compute output' + _ = self.call_component( + diagnostic, {'time': dt + timedelta(seconds=45)}) + assert diagnostic.times_called == 5, 'should re-compute output' + _ = self.call_component( + diagnostic, {'time': dt + timedelta(seconds=50)}) + assert diagnostic.times_called == 5, 'should not re-compute output' + + +class ImplicitTests(unittest.TestCase): + + component_class = MockImplicit + + def call_component(self, component, state): + return component(state, timedelta(seconds=1)) + + def test_empty_implicit(self): + implicit = self.component_class( + {}, {}, {}, {}, {}) + tendencies, diagnostics = self.call_component( + implicit, {'time': timedelta(seconds=0)}) + assert tendencies == {} + assert diagnostics == {} + assert len(implicit.state_given) == 1 + assert 'time' in implicit.state_given.keys() + assert implicit.state_given['time'] == timedelta(seconds=0) + assert implicit.times_called == 1 + + def test_timedelta_is_passed(self): + implicit = MockImplicit({}, {}, {}, {}, {}) + tendencies, diagnostics = implicit( + {'time': timedelta(seconds=0)}, timedelta(seconds=5)) + assert tendencies == {} + assert diagnostics == {} + assert implicit.timestep_given == timedelta(seconds=5) + assert implicit.times_called == 1 + + def test_input_no_transformations(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_properties = {} + output_properties = {} + diagnostic_output = {} + output_state = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state + ) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _, _ = self.call_component(implicit, state) + assert len(implicit.state_given) == 2 + assert 'time' in implicit.state_given.keys() + assert 'input1' in implicit.state_given.keys() + assert isinstance(implicit.state_given['input1'], np.ndarray) + assert np.all(implicit.state_given['input1'] == np.ones([10])) + + def test_input_converts_units(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_properties = {} + output_properties = {} + diagnostic_output = {} + output_state = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state + ) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'km'} + ) + } + _, _ = self.call_component(implicit, state) + assert len(implicit.state_given) == 2 + assert 'time' in implicit.state_given.keys() + assert 'input1' in implicit.state_given.keys() + assert isinstance(implicit.state_given['input1'], np.ndarray) + assert np.all(implicit.state_given['input1'] == np.ones([10])*1000.) + + def test_input_collects_one_dimension(self): + input_properties = { + 'input1': { + 'dims': ['*'], + 'units': 'm' + } + } + diagnostic_properties = {} + output_properties = {} + diagnostic_output = {} + output_state = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state + ) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _, _ = self.call_component(implicit, state) + assert len(implicit.state_given) == 2 + assert 'time' in implicit.state_given.keys() + assert 'input1' in implicit.state_given.keys() + assert isinstance(implicit.state_given['input1'], np.ndarray) + assert np.all(implicit.state_given['input1'] == np.ones([10])) + + def test_input_is_aliased(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'in1', + } + } + diagnostic_properties = {} + output_properties = {} + diagnostic_output = {} + output_state = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state + ) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _, _ = self.call_component(prognostic, state) + assert len(prognostic.state_given) == 2 + assert 'time' in prognostic.state_given.keys() + assert 'in1' in prognostic.state_given.keys() + assert isinstance(prognostic.state_given['in1'], np.ndarray) + assert np.all(prognostic.state_given['in1'] == np.ones([10])) + + def test_output_no_transformations(self): + input_properties = {} + diagnostic_properties = {} + output_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + }} + diagnostic_output = {} + output_state = { + 'output1': np.ones([10]), + } + prognostic = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state + ) + state = {'time': timedelta(0)} + _, output = self.call_component(prognostic, state) + assert len(output) == 1 + assert 'output1' in output.keys() + assert isinstance(output['output1'], DataArray) + assert len(output['output1'].dims) == 1 + assert 'dim1' in output['output1'].dims + assert 'units' in output['output1'].attrs + assert output['output1'].attrs['units'] == 'm/s' + assert np.all(output['output1'].values == np.ones([10])) + + def test_output_with_alias(self): + input_properties = {} + diagnostic_properties = {} + output_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s', + 'alias': 'out1', + }} + diagnostic_output = {} + output_state = { + 'out1': np.ones([10]), + } + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state + ) + state = {'time': timedelta(0)} + _, output = self.call_component(implicit, state) + assert len(output) == 1 + assert 'output1' in output.keys() + assert isinstance(output['output1'], DataArray) + assert len(output['output1'].dims) == 1 + assert 'dim1' in output['output1'].dims + assert 'units' in output['output1'].attrs + assert output['output1'].attrs['units'] == 'm/s' + assert np.all(output['output1'].values == np.ones([10])) + + def test_output_with_alias_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'out1', + } + } + diagnostic_properties = {} + output_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s', + } + } + diagnostic_output = {} + output_state = { + 'out1': np.ones([10]), + } + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _, output = self.call_component(implicit, state) + assert len(output) == 1 + assert 'output1' in output.keys() + assert isinstance(output['output1'], DataArray) + assert len(output['output1'].dims) == 1 + assert 'dim1' in output['output1'].dims + assert 'units' in output['output1'].attrs + assert output['output1'].attrs['units'] == 'm/s' + assert np.all(output['output1'].values == np.ones([10])) + + def test_output_with_dims_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + diagnostic_properties = {} + output_properties = { + 'output1': { + 'units': 'm/s', + } + } + diagnostic_output = {} + output_state = { + 'output1': np.ones([10]), + } + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _, output = self.call_component(implicit, state) + assert len(output) == 1 + assert 'output1' in output.keys() + assert isinstance(output['output1'], DataArray) + assert len(output['output1'].dims) == 1 + assert 'dim1' in output['output1'].dims + assert 'units' in output['output1'].attrs + assert output['output1'].attrs['units'] == 'm/s' + assert np.all(output['output1'].values == np.ones([10])) + + def test_diagnostics_no_transformations(self): + input_properties = {} + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + output_properties = {} + diagnostic_output = { + 'output1': np.ones([10]), + } + output_state = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state + ) + state = {'time': timedelta(0)} + diagnostics, _ = self.call_component(implicit, state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_with_alias(self): + input_properties = {} + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'out1', + } + } + output_properties = {} + diagnostic_output = { + 'out1': np.ones([10]), + } + output_state = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state + ) + state = {'time': timedelta(0)} + diagnostics, _ = self.call_component(implicit, state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_with_alias_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'out1', + } + } + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + output_properties = {} + diagnostic_output = { + 'out1': np.ones([10]), + } + output_state = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + diagnostics, _ = self.call_component(implicit, state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_with_dims_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + diagnostic_properties = { + 'output1': { + 'units': 'm', + } + } + output_properties = {} + diagnostic_output = { + 'output1': np.ones([10]), + } + output_state = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + diagnostics, _ = self.call_component(implicit, state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_input_scaling(self): + input_scale_factors = {'input1': 2.} + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_properties = {} + output_properties = {} + diagnostic_output = {} + output_state = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state, + input_scale_factors=input_scale_factors + ) + assert implicit.output_scale_factors == {} + assert implicit.diagnostic_scale_factors == {} + assert len(implicit.input_scale_factors) == 1 + assert 'input1' in implicit.input_scale_factors.keys() + assert implicit.input_scale_factors['input1'] == 2. + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + _, _ = self.call_component(implicit, state) + assert len(implicit.state_given) == 2 + assert 'time' in implicit.state_given.keys() + assert 'input1' in implicit.state_given.keys() + assert isinstance(implicit.state_given['input1'], np.ndarray) + assert np.all(implicit.state_given['input1'] == np.ones([10]) * 2.) + + def test_output_scaling(self): + output_scale_factors = {'output1': 3.} + input_properties = {} + diagnostic_properties = {} + output_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + } + } + diagnostic_output = {} + output_state = { + 'output1': np.ones([10]), + } + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state, + output_scale_factors=output_scale_factors, + ) + assert implicit.input_scale_factors == {} + assert implicit.diagnostic_scale_factors == {} + assert len(implicit.output_scale_factors) == 1 + assert 'output1' in implicit.output_scale_factors.keys() + assert implicit.output_scale_factors['output1'] == 3. + state = {'time': timedelta(0)} + _, output = self.call_component(implicit, state) + assert len(output) == 1 + assert 'output1' in output.keys() + assert isinstance(output['output1'], DataArray) + assert len(output['output1'].dims) == 1 + assert 'dim1' in output['output1'].dims + assert 'units' in output['output1'].attrs + assert output['output1'].attrs['units'] == 'm/s' + assert np.all(output['output1'].values == np.ones([10]) * 3.) + + def test_diagnostics_scaling(self): + diagnostic_scale_factors = {'output1': 0.} + input_properties = {} + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + output_properties = {} + diagnostic_output = { + 'output1': np.ones([10]), + } + tendency_output = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, tendency_output, + diagnostic_scale_factors=diagnostic_scale_factors, + ) + assert implicit.output_scale_factors == {} + assert implicit.input_scale_factors == {} + assert len(implicit.diagnostic_scale_factors) == 1 + assert 'output1' in implicit.diagnostic_scale_factors.keys() + assert implicit.diagnostic_scale_factors['output1'] == 0. + state = {'time': timedelta(0)} + diagnostics, _ = self.call_component(implicit, state) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10]) * 0.) + + def test_update_interval_on_timedelta(self): + input_properties = {} + diagnostic_properties = {} + output_properties = {} + diagnostic_output = {} + output_state = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state, + update_interval=timedelta(seconds=10) + ) + _, _ = self.call_component(implicit, {'time': timedelta(seconds=0)}) + assert implicit.times_called == 1 + _, _ = self.call_component(implicit, {'time': timedelta(seconds=0)}) + assert implicit.times_called == 1, 'should not re-compute output' + _, _ = self.call_component(implicit, {'time': timedelta(seconds=5)}) + assert implicit.times_called == 1, 'should not re-compute output' + _, _ = self.call_component(implicit, {'time': timedelta(seconds=10)}) + assert implicit.times_called == 2, 'should re-compute output' + _, _ = self.call_component(implicit, {'time': timedelta(seconds=15)}) + assert implicit.times_called == 2, 'should not re-compute output' + _, _ = self.call_component(implicit, {'time': timedelta(seconds=20)}) + assert implicit.times_called == 3, 'should re-compute output' + _, _ = self.call_component(implicit, {'time': timedelta(seconds=30)}) + assert implicit.times_called == 4, 'should re-compute output' + _, _ = self.call_component(implicit, {'time': timedelta(seconds=45)}) + assert implicit.times_called == 5, 'should re-compute output' + _, _ = self.call_component(implicit, {'time': timedelta(seconds=50)}) + assert implicit.times_called == 5, 'should not re-compute output' + + def test_update_interval_on_datetime(self): + input_properties = {} + diagnostic_properties = {} + output_properties = {} + diagnostic_output = {} + output_state = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state, + update_interval=timedelta(seconds=10) + ) + dt = datetime(2010, 1, 1) + _, _ = self.call_component( + implicit, {'time': dt + timedelta(seconds=0)}) + assert implicit.times_called == 1 + _, _ = self.call_component( + implicit, {'time': dt + timedelta(seconds=0)}) + assert implicit.times_called == 1, 'should not re-compute output' + _, _ = self.call_component( + implicit, {'time': dt + timedelta(seconds=5)}) + assert implicit.times_called == 1, 'should not re-compute output' + _, _ = self.call_component( + implicit, {'time': dt + timedelta(seconds=10)}) + assert implicit.times_called == 2, 'should re-compute output' + _, _ = self.call_component( + implicit, {'time': dt + timedelta(seconds=15)}) + assert implicit.times_called == 2, 'should not re-compute output' + _, _ = self.call_component( + implicit, {'time': dt + timedelta(seconds=20)}) + assert implicit.times_called == 3, 'should re-compute output' + _, _ = self.call_component( + implicit, {'time': dt + timedelta(seconds=30)}) + assert implicit.times_called == 4, 'should re-compute output' + _, _ = self.call_component( + implicit, {'time': dt + timedelta(seconds=45)}) + assert implicit.times_called == 5, 'should re-compute output' + _, _ = self.call_component( + implicit, {'time': dt + timedelta(seconds=50)}) + assert implicit.times_called == 5, 'should not re-compute output' + + def test_tendencies_in_diagnostics_no_tendency(self): + input_properties = {} + diagnostic_properties = {} + output_properties = {} + diagnostic_output = {} + output_state = {} + implicit = MockImplicit( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state, tendencies_in_diagnostics=True + ) + assert implicit.input_properties == {} + assert implicit.diagnostic_properties == {} + assert implicit.output_properties == {} + state = {'time': timedelta(0)} + diagnostics, _ = implicit(state, timedelta(seconds=5)) + assert diagnostics == {} + + def test_tendencies_in_diagnostics_one_tendency(self): + input_properties = {} + diagnostic_properties = {} + output_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_output = {} + output_state = { + 'output1': np.ones([10]) * 20., + } + implicit = MockImplicit( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state, tendencies_in_diagnostics=True, + ) + assert len(implicit.diagnostic_properties) == 1 + assert 'output1_tendency_from_mockimplicit' in implicit.diagnostic_properties.keys() + assert 'output1' in input_properties.keys(), 'Implicit needs original value to calculate tendency' + assert input_properties['output1']['dims'] == ['dim1'] + assert input_properties['output1']['units'] == 'm' + properties = implicit.diagnostic_properties[ + 'output1_tendency_from_mockimplicit'] + assert properties['dims'] == ['dim1'] + assert properties['units'] == 'm s^-1' + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10])*10., + dims=['dim1'], + attrs={'units': 'm'} + ), + } + diagnostics, _ = implicit(state, timedelta(seconds=5)) + assert 'output1_tendency_from_mockimplicit' in diagnostics.keys() + assert len( + diagnostics['output1_tendency_from_mockimplicit'].dims) == 1 + assert 'dim1' in diagnostics['output1_tendency_from_mockimplicit'] + assert diagnostics['output1_tendency_from_mockimplicit'].attrs['units'] == 'm s^-1' + assert np.all( + diagnostics['output1_tendency_from_mockimplicit'].values == 2.) + + def test_tendencies_in_diagnostics_one_tendency_mismatched_units(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'km' + } + } + diagnostic_properties = {} + output_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_output = {} + output_state = { + 'output1': np.ones([10]) * 20., + } + with self.assertRaises(InvalidPropertyDictError): + implicit = MockImplicit( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state, tendencies_in_diagnostics=True, + ) + + def test_tendencies_in_diagnostics_one_tendency_mismatched_dims(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_properties = {} + output_properties = { + 'output1': { + 'dims': ['dim2'], + 'units': 'm' + } + } + diagnostic_output = {} + output_state = { + 'output1': np.ones([10]) * 20., + } + with self.assertRaises(InvalidPropertyDictError): + implicit = MockImplicit( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state, tendencies_in_diagnostics=True, + ) + + def test_tendencies_in_diagnostics_one_tendency_with_component_name(self): + input_properties = {} + diagnostic_properties = {} + output_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_output = {} + output_state = { + 'output1': np.ones([10]) * 7., + } + implicit = MockImplicit( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state, tendencies_in_diagnostics=True, + name='component' + ) + assert len(implicit.diagnostic_properties) == 1 + assert 'output1_tendency_from_component' in implicit.diagnostic_properties.keys() + properties = implicit.diagnostic_properties[ + 'output1_tendency_from_component'] + assert properties['dims'] == ['dim1'] + assert properties['units'] == 'm s^-1' + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]) * 2., + dims=['dim1'], + attrs={'units': 'm'} + ), + } + diagnostics, _ = implicit(state, timedelta(seconds=5)) + assert 'output1_tendency_from_component' in diagnostics.keys() + assert len(diagnostics['output1_tendency_from_component'].dims) == 1 + assert 'dim1' in diagnostics['output1_tendency_from_component'] + assert diagnostics['output1_tendency_from_component'].attrs['units'] == 'm s^-1' + assert np.all(diagnostics['output1_tendency_from_component'].values == 1.) + if __name__ == '__main__': diff --git a/tests/test_composite.py b/tests/test_composite.py new file mode 100644 index 0000000..25b8017 --- /dev/null +++ b/tests/test_composite.py @@ -0,0 +1,295 @@ +import pytest +import mock +from sympl import ( + Prognostic, Diagnostic, Monitor, PrognosticComposite, DiagnosticComposite, + MonitorComposite, SharedKeyError, DataArray +) + +def same_list(list1, list2): + return (len(list1) == len(list2) and all( + [item in list2 for item in list1] + [item in list1 for item in list2])) + + +class MockPrognostic(Prognostic): + + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + + def __init__(self, **kwargs): + super(MockPrognostic, self).__init__(**kwargs) + + def array_call(self, state): + return {}, {} + + +class MockPrognostic2(Prognostic): + + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + + def __init__(self, **kwargs): + super(MockPrognostic, self).__init__(**kwargs) + + def array_call(self, state): + return {}, {} + + +class MockDiagnostic(Diagnostic): + + input_properties = {} + diagnostic_properties = {} + + def __init__(self, **kwargs): + super(MockDiagnostic, self).__init__(**kwargs) + + def array_call(self, state): + return {} + + + +class MockMonitor(Monitor): + + def store(self, state): + return + + +def test_empty_prognostic_composite(): + prognostic_composite = PrognosticComposite() + state = {'air_temperature': 273.15} + tendencies, diagnostics = prognostic_composite(state) + assert len(tendencies) == 0 + assert len(diagnostics) == 0 + assert isinstance(tendencies, dict) + assert isinstance(diagnostics, dict) + + +@mock.patch.object(MockPrognostic, '__call__') +def test_prognostic_composite_calls_one_prognostic(mock_call): + mock_call.return_value = ({'air_temperature': 0.5}, {'foo': 50.}) + prognostic_composite = PrognosticComposite(MockPrognostic()) + state = {'air_temperature': 273.15} + tendencies, diagnostics = prognostic_composite(state) + assert mock_call.called + assert tendencies == {'air_temperature': 0.5} + assert diagnostics == {'foo': 50.} + + +@mock.patch.object(MockPrognostic, '__call__') +def test_prognostic_composite_calls_two_prognostics(mock_call): + mock_call.return_value = ({'air_temperature': 0.5}, {}) + prognostic_composite = PrognosticComposite( + MockPrognostic(), MockPrognostic()) + state = {'air_temperature': 273.15} + tendencies, diagnostics = prognostic_composite(state) + assert mock_call.called + assert mock_call.call_count == 2 + assert tendencies == {'air_temperature': 1.} + assert diagnostics == {} + + +def test_empty_diagnostic_composite(): + diagnostic_composite = DiagnosticComposite() + state = {'air_temperature': 273.15} + diagnostics = diagnostic_composite(state) + assert len(diagnostics) == 0 + assert isinstance(diagnostics, dict) + + +@mock.patch.object(MockDiagnostic, '__call__') +def test_diagnostic_composite_calls_one_diagnostic(mock_call): + mock_call.return_value = {'foo': 50.} + diagnostic_composite = DiagnosticComposite(MockDiagnostic()) + state = {'air_temperature': 273.15} + diagnostics = diagnostic_composite(state) + assert mock_call.called + assert diagnostics == {'foo': 50.} + + +def test_empty_monitor_collection(): + # mainly we're testing that nothing errors + monitor_collection = MonitorComposite() + state = {'air_temperature': 273.15} + monitor_collection.store(state) + + +@mock.patch.object(MockMonitor, 'store') +def test_monitor_collection_calls_one_monitor(mock_store): + mock_store.return_value = None + monitor_collection = MonitorComposite(MockMonitor()) + state = {'air_temperature': 273.15} + monitor_collection.store(state) + assert mock_store.called + + +@mock.patch.object(MockMonitor, 'store') +def test_monitor_collection_calls_two_monitors(mock_store): + mock_store.return_value = None + monitor_collection = MonitorComposite(MockMonitor(), MockMonitor()) + state = {'air_temperature': 273.15} + monitor_collection.store(state) + assert mock_store.called + assert mock_store.call_count == 2 + + +def test_prognostic_composite_cannot_use_diagnostic(): + try: + PrognosticComposite(MockDiagnostic()) + except TypeError: + pass + except Exception as err: + raise err + else: + raise AssertionError('TypeError should have been raised') + + +def test_diagnostic_composite_cannot_use_prognostic(): + try: + DiagnosticComposite(MockPrognostic()) + except TypeError: + pass + except Exception as err: + raise err + else: + raise AssertionError('TypeError should have been raised') + + +@mock.patch.object(MockDiagnostic, '__call__') +def test_diagnostic_composite_call(mock_call): + mock_call.return_value = {'foo': 5.} + state = {'bar': 10.} + diagnostics = DiagnosticComposite(MockDiagnostic()) + new_state = diagnostics(state) + assert list(state.keys()) == ['bar'] + assert state['bar'] == 10. + assert list(new_state.keys()) == ['foo'] + assert new_state['foo'] == 5. + + +def test_prognostic_composite_includes_attributes(): + prognostic = MockPrognostic() + prognostic.input_properties = {'input1': {'units': 'K', 'dims':['z', 'y', 'x']}} + prognostic.diagnostic_properties = {'diagnostic1': {'units': 'm/s'}} + prognostic.tendency_properties = {'tendency1': {}} + composite = PrognosticComposite(prognostic) + assert same_list(composite.input_properties.keys(), ('input1',)) + assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1',)) + assert same_list(composite.tendency_properties.keys(), ('tendency1',)) + + +def test_prognostic_composite_includes_attributes_from_two(): + prognostic1 = MockPrognostic() + prognostic1.input_properties = {'input1': {'units': 'K', 'dims':['z', 'y', 'x']}} + prognostic1.diagnostic_properties = {'diagnostic1': {'units': 'm/s'}} + prognostic1.tendency_properties = {'tendency1': {}} + prognostic2 = MockPrognostic() + prognostic2.input_properties = {'input2': {'units': 'K', 'dims':['z', 'y', 'x']}} + prognostic2.diagnostic_properties = {'diagnostic2': {'units': 'm/s'}} + prognostic2.tendency_properties = {'tendency2': {}} + composite = PrognosticComposite(prognostic1, prognostic2) + assert same_list(composite.input_properties.keys(), ('input1', 'input2')) + assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) + assert same_list(composite.tendency_properties.keys(), ('tendency1', 'tendency2')) + + +def test_prognostic_merges_attributes(): + prognostic1 = MockPrognostic() + prognostic1.input_properties = {'input1': {}} + prognostic1.diagnostic_properties = {'diagnostic1': {}} + prognostic1.tendency_properties = {'tendency1': {}, 'tendency2': {}} + prognostic2 = MockPrognostic() + prognostic2.input_properties = {'input1': {}, 'input2': {}} + prognostic2.diagnostic_properties = {'diagnostic2': {}} + prognostic2.tendency_properties = {'tendency2': {}} + composite = PrognosticComposite(prognostic1, prognostic2) + assert same_list(composite.input_properties.keys(), ('input1', 'input2')) + assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) + assert same_list(composite.tendency_properties.keys(), ('tendency1', 'tendency2')) + + +def test_prognostic_composite_ensures_valid_state(): + prognostic1 = MockPrognostic() + prognostic1.input_properties = {'input1': {}} + prognostic1.diagnostic_properties = {'diagnostic1': {}} + prognostic2 = MockPrognostic() + prognostic2.input_properties = {'input1': {}, 'input2': {}} + prognostic2.diagnostic_properties = {'diagnostic1': {}} + try: + PrognosticComposite(prognostic1, prognostic2) + except SharedKeyError: + pass + except Exception as err: + raise err + else: + raise AssertionError( + 'Should not be able to have overlapping diagnostics in composite') + + +@mock.patch.object(MockPrognostic, '__call__') +@mock.patch.object(MockPrognostic2, '__call__') +def test_prognostic_component_handles_units_when_combining(mock_call, mock2_call): + mock_call.return_value = ({ + 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})}, {}) + mock2_call.return_value = ({ + 'eastward_wind': DataArray(50., attrs={'units': 'cm/s'})}, {}) + prognostic1 = MockPrognostic() + prognostic2 = MockPrognostic2() + composite = PrognosticComposite(prognostic1, prognostic2) + tendencies, diagnostics = composite({}) + assert tendencies['eastward_wind'].to_units('m/s').values.item() == 1.5 + + +def test_diagnostic_composite_includes_attributes(): + diagnostic = MockDiagnostic() + diagnostic.input_properties = {'input1': {}} + diagnostic.diagnostic_properties = {'diagnostic1': {}} + composite = DiagnosticComposite(diagnostic) + assert same_list(composite.input_properties.keys(), ('input1',)) + assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1',)) + + +def test_diagnostic_composite_includes_attributes_from_two(): + diagnostic1 = MockDiagnostic() + diagnostic1.input_properties = {'input1': {}} + diagnostic1.diagnostic_properties = {'diagnostic1': {}} + diagnostic2 = MockDiagnostic() + diagnostic2.input_properties = {'input2': {}} + diagnostic2.diagnostic_properties = {'diagnostic2': {}} + composite = DiagnosticComposite(diagnostic1, diagnostic2) + assert same_list(composite.input_properties.keys(), ('input1', 'input2')) + assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) + + +def test_diagnostic_composite_merges_attributes(): + diagnostic1 = MockDiagnostic() + diagnostic1.input_properties = {'input1': {}} + diagnostic1.diagnostic_properties = {'diagnostic1': {}} + diagnostic2 = MockDiagnostic() + diagnostic2.input_properties = {'input1': {}, 'input2': {}} + diagnostic2.diagnostic_properties = {'diagnostic2': {}} + composite = DiagnosticComposite(diagnostic1, diagnostic2) + assert same_list(composite.input_properties.keys(), ('input1', 'input2')) + assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) + + +def test_diagnostic_composite_ensures_valid_state(): + diagnostic1 = MockDiagnostic() + diagnostic1.input_properties = {'input1': {}} + diagnostic1.diagnostic_properties = {'diagnostic1': {}} + diagnostic2 = MockDiagnostic() + diagnostic2.input_properties = {'input1': {}, 'input2': {}} + diagnostic2.diagnostic_properties = {'diagnostic1': {}} + try: + DiagnosticComposite(diagnostic1, diagnostic2) + except SharedKeyError: + pass + except Exception as err: + raise err + else: + raise AssertionError( + 'Should not be able to have overlapping diagnostics in composite') + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/tests/test_get_restore_numpy_array.py b/tests/test_get_restore_numpy_array.py index 88cc04f..2233bdf 100644 --- a/tests/test_get_restore_numpy_array.py +++ b/tests/test_get_restore_numpy_array.py @@ -1076,6 +1076,26 @@ def setUp(self): def tearDown(self): set_direction_names(x=(), y=(), z=()) + def test_restores_with_dims(self): + raw_arrays = { + 'output1': np.ones([10]), + } + output_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + output = restore_data_arrays_with_properties( + raw_arrays, output_properties, {}, {}) + assert len(output) == 1 + assert 'output1' in output.keys() + assert isinstance(output['output1'], DataArray) + assert len(output['output1'].dims) == 1 + assert 'dim1' in output['output1'].dims + assert 'units' in output['output1'].attrs.keys() + assert output['output1'].attrs['units'] == 'm' + def test_returns_simple_value(self): input_state = { 'air_temperature': DataArray( From 3b481cce74abaa0caf0c655f724cb4437db3799f Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 21 Mar 2018 10:48:46 -0700 Subject: [PATCH 09/98] Missed file from last commit --- sympl/_core/util.py | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/sympl/_core/util.py b/sympl/_core/util.py index 08bf7a4..57dfeba 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -351,24 +351,36 @@ def restore_data_arrays_with_properties( 'requested output {} is not present in raw_arrays'.format( from_name)) array = raw_arrays[from_name] - from_dims = input_properties[dims_like]['dims'] - result_like = input_state[dims_like] - try: - out_dict[quantity_name] = restore_dimensions( - array, - from_dims=from_dims, - result_like=result_like, - result_attrs=attrs) - except ShapeMismatchError: - raise InvalidPropertyDictError( - 'output quantity {} has dims_like input {}, but the ' - 'provided output array for {} has ' - 'a shape {} incompatible with the input shape {} of {}. ' - 'Do they really have the same dimensions?'.format( - quantity_name, dims_like, quantity_name, array.shape, - result_like.shape, dims_like + if dims_like in input_properties.keys(): + from_dims = input_properties[dims_like]['dims'] + result_like = input_state[dims_like] + try: + out_dict[quantity_name] = restore_dimensions( + array, + from_dims=from_dims, + result_like=result_like, + result_attrs=attrs) + except ShapeMismatchError: + raise InvalidPropertyDictError( + 'output quantity {} has dims_like input {}, but the ' + 'provided output array for {} has ' + 'a shape {} incompatible with the input shape {} of {}. ' + 'Do they really have the same dimensions?'.format( + quantity_name, dims_like, quantity_name, array.shape, + result_like.shape, dims_like + ) ) + elif 'dims' in properties.keys(): + out_dict[quantity_name] = DataArray( + array, + dims=properties['dims'], + attrs={'units': properties['units']}, ) + else: + raise InvalidPropertyDictError( + 'Could not determine dimensions for {}, make sure dims_like or ' + 'dims is specified in its property dictionary'.format( + quantity_name)) return out_dict From 7a11348bfd12898aa6aa9053b575b4b636548478 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 21 Mar 2018 12:21:56 -0700 Subject: [PATCH 10/98] Updated code and tests to pass * Mainly changes to TimeSteppers and test_timestepping * TimeDifferencingWrapper is now in components rather than core --- HISTORY.rst | 2 + sympl/__init__.py | 4 +- sympl/_components/__init__.py | 7 +- sympl/_components/basic.py | 59 +++- sympl/_components/timesteppers.py | 28 +- sympl/_core/base_components.py | 2 +- sympl/_core/composite.py | 31 +- sympl/_core/timestepper.py | 12 +- sympl/_core/wrappers.py | 58 ---- tests/test_base_components.py | 5 +- tests/test_composite.py | 78 +---- tests/test_time_differencing_wrapper.py | 144 +++++++++ tests/test_timestepping.py | 285 ++++++++---------- tests/test_wrapper.py | 384 ------------------------ 14 files changed, 379 insertions(+), 720 deletions(-) delete mode 100644 sympl/_core/wrappers.py create mode 100644 tests/test_time_differencing_wrapper.py delete mode 100644 tests/test_wrapper.py diff --git a/HISTORY.rst b/HISTORY.rst index 0bc1835..3151928 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,8 @@ Latest * Added a check for netcdftime having the required objects, to fall back on not using netcdftime when those are missing. This is because most objects are missing in older versions of netcdftime (that come packaged with netCDF4) (closes #23). +* TimeSteppers should now be called with individual Prognostics as args, rather + than a list of components, and will emit a warning when lists are given. Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/__init__.py b/sympl/__init__.py index 21bd4f4..bcd3419 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -19,11 +19,11 @@ restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, set_direction_names, add_direction_names) -from ._core.wrappers import TimeDifferencingWrapper from ._core.testing import ComponentTestBase from ._components import ( PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, - ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic) + ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, + TimeDifferencingWrapper) from ._core.time import datetime, timedelta __version__ = '0.3.1' diff --git a/sympl/_components/__init__.py b/sympl/_components/__init__.py index a818515..08ff312 100644 --- a/sympl/_components/__init__.py +++ b/sympl/_components/__init__.py @@ -1,8 +1,11 @@ from .netcdf import NetCDFMonitor, RestartMonitor from .plot import PlotFunctionMonitor -from .basic import ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic +from .basic import ( + ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, + TimeDifferencingWrapper) __all__ = ( PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, - ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic) + ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, + TimeDifferencingWrapper) diff --git a/sympl/_components/basic.py b/sympl/_components/basic.py index b1be2d2..f55c007 100644 --- a/sympl/_components/basic.py +++ b/sympl/_components/basic.py @@ -1,4 +1,5 @@ -from .._core.base_components import Prognostic, Diagnostic +from .._core.array import DataArray +from .._core.base_components import ImplicitPrognostic, Prognostic, Diagnostic from .._core.array import DataArray from .._core.units import unit_registry as ureg @@ -181,3 +182,59 @@ def __call__(self, state): ) } return tendencies, {} + + +class TimeDifferencingWrapper(ImplicitPrognostic): + """ + Wraps an Implicit object and turns it into an ImplicitPrognostic by applying + simple first-order time differencing to determine tendencies. + + Example + ------- + This how the wrapper should be used on an Implicit class + called GridScaleCondensation. + + >>> component = TimeDifferencingWrapper(GridScaleCondensation()) + """ + + def __init__(self, implicit): + self._implicit = implicit + + def __call__(self, state, timestep): + diagnostics, new_state = self._implicit(state, timestep) + tendencies = {} + timestep_seconds = timestep.total_seconds() + for varname, data_array in new_state.items(): + if isinstance(data_array, DataArray): + if varname in self._implicit.output_properties.keys(): + if varname not in state.keys(): + raise RuntimeError( + 'Cannot calculate tendency for {} because it is not' + ' present in the input state.'.format(varname)) + tendency = (data_array - state[varname].to_units(data_array.attrs['units'])) / timestep_seconds + if data_array.attrs['units'] == '': + tendency.attrs['units'] = 's^-1' + else: + tendency.attrs['units'] = data_array.attrs['units'] + ' s^-1' + tendencies[varname] = tendency.to_units( + self._implicit.output_properties[varname]['units'] + ' s^-1') + elif varname != 'time': + raise ValueError( + 'Wrapped implicit gave an output {} of type {}, but should' + 'only give sympl.DataArray objects.'.format( + varname, type(data_array))) + return tendencies, diagnostics + + @property + def tendencies(self): + return list(self.tendency_properties.keys()) + + @property + def tendency_properties(self): + return_dict = self._implicit.output_properties.copy() + return_dict.update(self._tendency_diagnostic_properties) + return return_dict + + def __getattr__(self, item): + if item not in ('outputs', 'output_properties'): + return getattr(self._implicit, item) diff --git a/sympl/_components/timesteppers.py b/sympl/_components/timesteppers.py index 6b08990..d209561 100644 --- a/sympl/_components/timesteppers.py +++ b/sympl/_components/timesteppers.py @@ -10,14 +10,14 @@ class SSPRungeKutta(TimeStepper): as proposed by Shu and Osher (1988). """ - def __init__(self, prognostic_list, stages=3): + def __init__(self, *args, stages=3): """ Initialize a strong stability preserving Runge-Kutta time stepper. Args ---- - prognostic_list : iterable of Prognostic - Objects used to get tendencies for time stepping. + *args : Prognostic + Objects to call for tendencies when doing time stepping. stages: int Number of stages to use. Should be 2 or 3. """ @@ -25,8 +25,8 @@ def __init__(self, prognostic_list, stages=3): raise ValueError( 'stages must be one of 2 or 3, received {}'.format(stages)) self._stages = stages - self._euler_stepper = AdamsBashforth(prognostic_list, order=1) - super(SSPRungeKutta, self).__init__(prognostic_list) + self._euler_stepper = AdamsBashforth(*args, order=1) + super(SSPRungeKutta, self).__init__(*args) def __call__(self, state, timestep): @@ -75,14 +75,14 @@ def _step_2_stages(self, state, timestep): class AdamsBashforth(TimeStepper): """A TimeStepper using the Adams-Bashforth scheme.""" - def __init__(self, prognostic_list, order=3): + def __init__(self, *args, order=3): """ Initialize an Adams-Bashforth time stepper. Args ---- - prognostic_list : iterable of Prognostic - Objects used to get tendencies for time stepping. + *args : Prognostic + Objects to call for tendencies when doing time stepping. order : int, optional The order of accuracy to use. Must be between 1 and 4. 1 is the same as the Euler method. Default is 3. @@ -96,7 +96,7 @@ def __init__(self, prognostic_list, order=3): self._order = order self._timestep = None self._tendencies_list = [] - super(AdamsBashforth, self).__init__(prognostic_list) + super(AdamsBashforth, self).__init__(*args) def __call__(self, state, timestep): """ @@ -184,16 +184,14 @@ class Leapfrog(TimeStepper): $t_{n+1}$, according to Williams (2009)) closer to the mean of the values at $t_{n-1}$ and $t_{n+1}$.""" - def __init__( - self, prognostic_list, asselin_strength=0.05, - alpha=0.5): + def __init__(self, *args, asselin_strength=0.05, alpha=0.5): """ Initialize a Leapfrog time stepper. Args ---- - prognostic_list : iterable of Prognostic - Objects used to get tendencies for time stepping. + *args : Prognostic + Objects to call for tendencies when doing time stepping. asselin_strength : float, optional The filter parameter used to determine the strength of the Asselin filter. Default is 0.05. @@ -215,7 +213,7 @@ def __init__( self._asselin_strength = asselin_strength self._timestep = None self._alpha = alpha - super(Leapfrog, self).__init__(prognostic_list) + super(Leapfrog, self).__init__(*args) def __call__(self, state, timestep): """ diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index b394276..c61fdd9 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -1,5 +1,5 @@ import abc -from util import ( +from .util import ( get_numpy_arrays_with_properties, restore_data_arrays_with_properties) from .time import timedelta from .exceptions import InvalidPropertyDictError diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index c43876e..cd19a33 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -41,12 +41,17 @@ def __init__(self, *args): if self.component_class is not None: ensure_components_have_class(args, self.component_class) self.component_list = args - if hasattr(self, 'diagnostics'): - if (len(self.diagnostics) != - sum([len(comp.diagnostics) for comp in self.component_list])): - raise SharedKeyError( - 'Two components in a composite should not compute ' - 'the same diagnostic') + if hasattr(self.component_class, 'diagnostic_properties'): + diagnostic_names = [] + for component in self.component_list: + diagnostic_names.extend(component.diagnostic_properties.keys()) + print(diagnostic_names) + for name in diagnostic_names: + if diagnostic_names.count(name) > 1: + raise SharedKeyError( + 'Two components in a composite should not compute ' + 'the same diagnostic, but multiple passed ' + 'components compute {}'.format(name)) def _combine_attribute(self, attr): return_attr = [] @@ -57,16 +62,10 @@ def _combine_attribute(self, attr): def ensure_components_have_class(components, component_class): for component in components: - for attr in ('input_properties', 'output_properties', - 'diagnostic_properties', 'tendency_properties'): - if hasattr(component_class, attr) and not hasattr(component, attr): - raise TypeError( - 'Component should have attribute {} but does not'.format( - attr)) - elif hasattr(component, attr) and not hasattr(component_class, attr): - raise TypeError( - 'Component should not have attribute {}, but does'.format( - attr)) + if not isinstance(component, component_class): + raise TypeError( + 'Component should be of type {} but is type {}'.format( + component_class, type(component))) class PrognosticComposite(ComponentComposite): diff --git a/sympl/_core/timestepper.py b/sympl/_core/timestepper.py index ae87c6e..969c5ea 100644 --- a/sympl/_core/timestepper.py +++ b/sympl/_core/timestepper.py @@ -1,6 +1,7 @@ import abc from .composite import PrognosticComposite from .time import timedelta +import warnings class TimeStepper(object): @@ -73,20 +74,25 @@ def __repr__(self): self._making_repr = False return return_value - def __init__(self, prognostic_list, tendencies_in_diagnostics=False): + def __init__(self, *args, tendencies_in_diagnostics=False): """ Initialize the TimeStepper. Parameters ---------- - prognostic_list : list of Prognostic + *args : Prognostic Objects to call for tendencies when doing time stepping. tendencies_in_diagnostics : bool, optional A boolean indicating whether this object will put tendencies of quantities in its diagnostic output. """ + if len(args) == 1 and isinstance(args[0], list): + warnings.warn( + 'TimeSteppers should be given individual Prognostics rather ' + 'than a list, and will not accept lists in a later version.') + args = args[0] self._tendencies_in_diagnostics = tendencies_in_diagnostics - self.prognostic = PrognosticComposite(prognostic_list) + self.prognostic = PrognosticComposite(*args) if tendencies_in_diagnostics: self._insert_tendencies_to_diagnostic_properties() diff --git a/sympl/_core/wrappers.py b/sympl/_core/wrappers.py deleted file mode 100644 index cae0e00..0000000 --- a/sympl/_core/wrappers.py +++ /dev/null @@ -1,58 +0,0 @@ -from .base_components import ImplicitPrognostic -from .array import DataArray - - -class TimeDifferencingWrapper(ImplicitPrognostic): - """ - Wraps an Implicit object and turns it into an ImplicitPrognostic by applying - simple first-order time differencing to determine tendencies. - - Example - ------- - This how the wrapper should be used on an Implicit class - called GridScaleCondensation. - - >>> component = TimeDifferencingWrapper(GridScaleCondensation()) - """ - - def __init__(self, implicit): - self._implicit = implicit - - def __call__(self, state, timestep): - diagnostics, new_state = self._implicit(state, timestep) - tendencies = {} - timestep_seconds = timestep.total_seconds() - for varname, data_array in new_state.items(): - if isinstance(data_array, DataArray): - if varname in self._implicit.output_properties.keys(): - if varname not in state.keys(): - raise RuntimeError( - 'Cannot calculate tendency for {} because it is not' - ' present in the input state.'.format(varname)) - tendency = (data_array - state[varname].to_units(data_array.attrs['units'])) / timestep_seconds - if data_array.attrs['units'] == '': - tendency.attrs['units'] = 's^-1' - else: - tendency.attrs['units'] = data_array.attrs['units'] + ' s^-1' - tendencies[varname] = tendency.to_units( - self._implicit.output_properties[varname]['units'] + ' s^-1') - elif varname != 'time': - raise ValueError( - 'Wrapped implicit gave an output {} of type {}, but should' - 'only give sympl.DataArray objects.'.format( - varname, type(data_array))) - return tendencies, diagnostics - - @property - def tendencies(self): - return list(self.tendency_properties.keys()) - - @property - def tendency_properties(self): - return_dict = self._implicit.output_properties.copy() - return_dict.update(self._tendency_diagnostic_properties) - return return_dict - - def __getattr__(self, item): - if item not in ('outputs', 'output_properties'): - return getattr(self._implicit, item) diff --git a/tests/test_base_components.py b/tests/test_base_components.py index 0e3e3f9..561c9a9 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -1764,7 +1764,7 @@ def test_tendencies_in_diagnostics_one_tendency(self): assert 'output1_tendency_from_mockimplicit' in diagnostics.keys() assert len( diagnostics['output1_tendency_from_mockimplicit'].dims) == 1 - assert 'dim1' in diagnostics['output1_tendency_from_mockimplicit'] + assert 'dim1' in diagnostics['output1_tendency_from_mockimplicit'].dims assert diagnostics['output1_tendency_from_mockimplicit'].attrs['units'] == 'm s^-1' assert np.all( diagnostics['output1_tendency_from_mockimplicit'].values == 2.) @@ -1852,11 +1852,10 @@ def test_tendencies_in_diagnostics_one_tendency_with_component_name(self): diagnostics, _ = implicit(state, timedelta(seconds=5)) assert 'output1_tendency_from_component' in diagnostics.keys() assert len(diagnostics['output1_tendency_from_component'].dims) == 1 - assert 'dim1' in diagnostics['output1_tendency_from_component'] + assert 'dim1' in diagnostics['output1_tendency_from_component'].dims assert diagnostics['output1_tendency_from_component'].attrs['units'] == 'm s^-1' assert np.all(diagnostics['output1_tendency_from_component'].values == 1.) - if __name__ == '__main__': pytest.main([__file__]) diff --git a/tests/test_composite.py b/tests/test_composite.py index 25b8017..775b3aa 100644 --- a/tests/test_composite.py +++ b/tests/test_composite.py @@ -5,6 +5,7 @@ MonitorComposite, SharedKeyError, DataArray ) + def same_list(list1, list2): return (len(list1) == len(list2) and all( [item in list2 for item in list1] + [item in list1 for item in list2])) @@ -30,7 +31,7 @@ class MockPrognostic2(Prognostic): tendency_properties = {} def __init__(self, **kwargs): - super(MockPrognostic, self).__init__(**kwargs) + super(MockPrognostic2, self).__init__(**kwargs) def array_call(self, state): return {}, {} @@ -48,7 +49,6 @@ def array_call(self, state): return {} - class MockMonitor(Monitor): def store(self, state): @@ -167,47 +167,6 @@ def test_diagnostic_composite_call(mock_call): assert new_state['foo'] == 5. -def test_prognostic_composite_includes_attributes(): - prognostic = MockPrognostic() - prognostic.input_properties = {'input1': {'units': 'K', 'dims':['z', 'y', 'x']}} - prognostic.diagnostic_properties = {'diagnostic1': {'units': 'm/s'}} - prognostic.tendency_properties = {'tendency1': {}} - composite = PrognosticComposite(prognostic) - assert same_list(composite.input_properties.keys(), ('input1',)) - assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1',)) - assert same_list(composite.tendency_properties.keys(), ('tendency1',)) - - -def test_prognostic_composite_includes_attributes_from_two(): - prognostic1 = MockPrognostic() - prognostic1.input_properties = {'input1': {'units': 'K', 'dims':['z', 'y', 'x']}} - prognostic1.diagnostic_properties = {'diagnostic1': {'units': 'm/s'}} - prognostic1.tendency_properties = {'tendency1': {}} - prognostic2 = MockPrognostic() - prognostic2.input_properties = {'input2': {'units': 'K', 'dims':['z', 'y', 'x']}} - prognostic2.diagnostic_properties = {'diagnostic2': {'units': 'm/s'}} - prognostic2.tendency_properties = {'tendency2': {}} - composite = PrognosticComposite(prognostic1, prognostic2) - assert same_list(composite.input_properties.keys(), ('input1', 'input2')) - assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) - assert same_list(composite.tendency_properties.keys(), ('tendency1', 'tendency2')) - - -def test_prognostic_merges_attributes(): - prognostic1 = MockPrognostic() - prognostic1.input_properties = {'input1': {}} - prognostic1.diagnostic_properties = {'diagnostic1': {}} - prognostic1.tendency_properties = {'tendency1': {}, 'tendency2': {}} - prognostic2 = MockPrognostic() - prognostic2.input_properties = {'input1': {}, 'input2': {}} - prognostic2.diagnostic_properties = {'diagnostic2': {}} - prognostic2.tendency_properties = {'tendency2': {}} - composite = PrognosticComposite(prognostic1, prognostic2) - assert same_list(composite.input_properties.keys(), ('input1', 'input2')) - assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) - assert same_list(composite.tendency_properties.keys(), ('tendency1', 'tendency2')) - - def test_prognostic_composite_ensures_valid_state(): prognostic1 = MockPrognostic() prognostic1.input_properties = {'input1': {}} @@ -240,39 +199,6 @@ def test_prognostic_component_handles_units_when_combining(mock_call, mock2_call assert tendencies['eastward_wind'].to_units('m/s').values.item() == 1.5 -def test_diagnostic_composite_includes_attributes(): - diagnostic = MockDiagnostic() - diagnostic.input_properties = {'input1': {}} - diagnostic.diagnostic_properties = {'diagnostic1': {}} - composite = DiagnosticComposite(diagnostic) - assert same_list(composite.input_properties.keys(), ('input1',)) - assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1',)) - - -def test_diagnostic_composite_includes_attributes_from_two(): - diagnostic1 = MockDiagnostic() - diagnostic1.input_properties = {'input1': {}} - diagnostic1.diagnostic_properties = {'diagnostic1': {}} - diagnostic2 = MockDiagnostic() - diagnostic2.input_properties = {'input2': {}} - diagnostic2.diagnostic_properties = {'diagnostic2': {}} - composite = DiagnosticComposite(diagnostic1, diagnostic2) - assert same_list(composite.input_properties.keys(), ('input1', 'input2')) - assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) - - -def test_diagnostic_composite_merges_attributes(): - diagnostic1 = MockDiagnostic() - diagnostic1.input_properties = {'input1': {}} - diagnostic1.diagnostic_properties = {'diagnostic1': {}} - diagnostic2 = MockDiagnostic() - diagnostic2.input_properties = {'input1': {}, 'input2': {}} - diagnostic2.diagnostic_properties = {'diagnostic2': {}} - composite = DiagnosticComposite(diagnostic1, diagnostic2) - assert same_list(composite.input_properties.keys(), ('input1', 'input2')) - assert same_list(composite.diagnostic_properties.keys(), ('diagnostic1', 'diagnostic2')) - - def test_diagnostic_composite_ensures_valid_state(): diagnostic1 = MockDiagnostic() diagnostic1.input_properties = {'input1': {}} diff --git a/tests/test_time_differencing_wrapper.py b/tests/test_time_differencing_wrapper.py new file mode 100644 index 0000000..cb36f83 --- /dev/null +++ b/tests/test_time_differencing_wrapper.py @@ -0,0 +1,144 @@ +from datetime import timedelta, datetime +import unittest +from sympl import ( + Prognostic, Implicit, Diagnostic, TimeDifferencingWrapper, DataArray +) +import pytest +from numpy.testing import assert_allclose +from copy import deepcopy + + +class MockPrognostic(Prognostic): + + def __init__(self): + self._num_updates = 0 + + def __call__(self, state): + self._num_updates += 1 + return {}, {'num_updates': self._num_updates} + + +class MockImplicit(Implicit): + + output_properties = { + 'value': { + 'dims': [], + 'units': 'm' + } + } + + diagnostic_properties = { + 'num_updates': { + 'dims': [], + 'units': '' + } + } + + def __init__(self): + self._num_updates = 0 + + def __call__(self, state, timestep): + self._num_updates += 1 + + return ( + {'num_updates': DataArray([self._num_updates], attrs={'units': ''})}, + {'value': DataArray([1], attrs={'units': 'm'})}) + + +class MockImplicitThatExpects(Implicit): + + input_properties = {'expected_field': {}} + output_properties = {'expected_field': {}} + diagnostic_properties = {'expected_field': {}} + + def __init__(self, expected_value): + self._expected_value = expected_value + + def __call__(self, state, timestep): + + input_value = state['expected_field'] + if input_value != self._expected_value: + raise ValueError( + 'Expected {}, but got {}'.format(self._expected_value, input_value)) + + return deepcopy(state), state + + +class MockPrognosticThatExpects(Prognostic): + + input_properties = {'expected_field': {}} + tendency_properties = {'expected_field': {}} + diagnostic_properties = {'expected_field': {}} + + def __init__(self, expected_value): + self._expected_value = expected_value + + def __call__(self, state): + + input_value = state['expected_field'] + if input_value != self._expected_value: + raise ValueError( + 'Expected {}, but got {}'.format(self._expected_value, input_value)) + + return deepcopy(state), state + + +class MockDiagnosticThatExpects(Diagnostic): + + input_properties = {'expected_field': {}} + diagnostic_properties = {'expected_field': {}} + + def __init__(self, expected_value): + self._expected_value = expected_value + + def __call__(self, state): + + input_value = state['expected_field'] + if input_value != self._expected_value: + raise ValueError( + 'Expected {}, but got {}'.format(self._expected_value, input_value)) + + return state + + +class TimeDifferencingTests(unittest.TestCase): + + def setUp(self): + self.implicit = MockImplicit() + self.wrapped = TimeDifferencingWrapper(self.implicit) + self.state = { + 'value': DataArray([0], attrs={'units': 'm'}) + } + + def tearDown(self): + self.component = None + + def testWrapperCallsImplicit(self): + tendencies, diagnostics = self.wrapped(self.state, timedelta(seconds=1)) + assert diagnostics['num_updates'].values[0] == 1 + tendencies, diagnostics = self.wrapped(self.state, timedelta(seconds=1)) + assert diagnostics['num_updates'].values[0] == 2 + assert len(diagnostics.keys()) == 1 + + def testWrapperComputesTendency(self): + tendencies, diagnostics = self.wrapped(self.state, timedelta(seconds=1)) + assert len(tendencies.keys()) == 1 + assert 'value' in tendencies.keys() + assert isinstance(tendencies['value'], DataArray) + assert_allclose(tendencies['value'].to_units('m s^-1').values[0], 1.) + assert_allclose(tendencies['value'].values[0], 1.) + + def testWrapperComputesTendencyWithUnitConversion(self): + state = { + 'value': DataArray([0.011], attrs={'units': 'km'}) + } + tendencies, diagnostics = self.wrapped(state, timedelta(seconds=5)) + assert 'value' in tendencies.keys() + assert isinstance(tendencies['value'], DataArray) + assert_allclose(tendencies['value'].to_units('m s^-1').values[0], -2) + assert_allclose(tendencies['value'].values[0], -2.) + assert_allclose(tendencies['value'].to_units('km s^-1').values[0], -0.002) + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index 388cde9..eafaccc 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -1,8 +1,7 @@ import pytest import mock from sympl import ( - Prognostic, Leapfrog, AdamsBashforth, DataArray, SSPRungeKutta) -from datetime import timedelta + Prognostic, Leapfrog, AdamsBashforth, DataArray, SSPRungeKutta, timedelta) import numpy as np @@ -13,7 +12,11 @@ def same_list(list1, list2): class MockPrognostic(Prognostic): - def __call__(self, state): + input_properties = {} + tendency_properties = {} + diagnostic_properties = {} + + def array_call(self, state): return {}, {} @@ -22,210 +25,174 @@ class TimesteppingBase(object): timestepper_class = None def test_unused_quantities_carried_over(self): - state = {'air_temperature': 273.} - time_stepper = self.timestepper_class([MockPrognostic()]) + state = {'time': timedelta(0), 'air_temperature': 273.} + time_stepper = self.timestepper_class(MockPrognostic()) timestep = timedelta(seconds=1.) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'air_temperature': 273.} - assert new_state == {'air_temperature': 273.} - - def test_timestepper_reveals_inputs(self): - prog1 = MockPrognostic() - prog1.input_properties = {'input1': {}} - time_stepper = self.timestepper_class([prog1]) - assert same_list(time_stepper.input_properties.keys(), ('input1',)) - - def test_timestepper_combines_inputs(self): - prog1 = MockPrognostic() - prog1.input_properties = {'input1': {}} - prog2 = MockPrognostic() - prog2.input_properties = {'input2': {}} - time_stepper = self.timestepper_class([prog1, prog2]) - assert same_list(time_stepper.input_properties.keys(), ('input1', 'input2')) + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} - def test_timestepper_doesnt_duplicate_inputs(self): + def test_timestepper_reveals_prognostics(self): prog1 = MockPrognostic() prog1.input_properties = {'input1': {}} - prog2 = MockPrognostic() - prog2.input_properties = {'input1': {}} - time_stepper = self.timestepper_class([prog1, prog2]) - assert same_list(time_stepper.input_properties.keys(), ('input1',)) - - def test_timestepper_reveals_outputs(self): - prog1 = MockPrognostic() - prog1.tendency_properties = {'output1': {}} - time_stepper = self.timestepper_class([prog1]) - assert same_list(time_stepper.output_properties.keys(), ('output1',)) - - def test_timestepper_combines_outputs(self): - prog1 = MockPrognostic() - prog1.tendency_properties = {'output1': {}} - prog2 = MockPrognostic() - prog2.tendency_properties = {'output2': {}} - time_stepper = self.timestepper_class([prog1, prog2]) - assert same_list(time_stepper.output_properties.keys(), ('output1', 'output2')) - - def test_timestepper_doesnt_duplicate_outputs(self): - prog1 = MockPrognostic() - prog1.tendency_properties = {'output1': {}} - prog2 = MockPrognostic() - prog2.tendency_properties = {'output1': {}} - time_stepper = self.timestepper_class([prog1, prog2]) - assert same_list(time_stepper.output_properties.keys(), ('output1',)) + time_stepper = self.timestepper_class(prog1) + assert same_list(time_stepper.prognostic_list, (prog1,)) @mock.patch.object(MockPrognostic, '__call__') def test_float_no_change_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) - state = {'air_temperature': 273.} + state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'air_temperature': 273.} - assert new_state == {'air_temperature': 273.} + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} assert diagnostics == {} @mock.patch.object(MockPrognostic, '__call__') def test_float_no_change_one_step_diagnostic(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': 0.}, {'foo': 'bar'}) - state = {'air_temperature': 273.} + state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'air_temperature': 273.} - assert new_state == {'air_temperature': 273.} + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} assert diagnostics == {'foo': 'bar'} @mock.patch.object(MockPrognostic, '__call__') def test_float_no_change_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) - state = {'air_temperature': 273.} + state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'air_temperature': 273.} - assert new_state == {'air_temperature': 273.} + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} state = new_state diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'air_temperature': 273.} - assert new_state == {'air_temperature': 273.} + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} state = new_state diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'air_temperature': 273.} - assert new_state == {'air_temperature': 273.} + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} @mock.patch.object(MockPrognostic, '__call__') def test_float_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 1.}, {}) - state = {'air_temperature': 273.} + state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'air_temperature': 273.} - assert new_state == {'air_temperature': 274.} + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 274.} @mock.patch.object(MockPrognostic, '__call__') def test_float_one_step_with_units(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'eastward_wind': DataArray(0.02, attrs={'units': 'km/s^2'})}, {}) - state = {'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} + state = {'time': timedelta(0), 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} - assert new_state == {'eastward_wind': DataArray(21., attrs={'units': 'm/s'})} + assert state == {'time': timedelta(0), 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} + assert same_list(new_state.keys(), ['time', 'eastward_wind']) + assert np.allclose(new_state['eastward_wind'].values, 21.) + assert new_state['eastward_wind'].attrs['units'] == 'm/s' @mock.patch.object(MockPrognostic, '__call__') def test_float_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 1.}, {}) - state = {'air_temperature': 273.} + state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'air_temperature': 273.} - assert new_state == {'air_temperature': 274.} + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 274.} state = new_state diagnostics, new_state = time_stepper.__call__(new_state, timestep) - assert state == {'air_temperature': 274.} - assert new_state == {'air_temperature': 275.} + assert state == {'time': timedelta(0), 'air_temperature': 274.} + assert new_state == {'time': timedelta(0), 'air_temperature': 275.} state = new_state diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'air_temperature': 275.} - assert new_state == {'air_temperature': 276.} + assert state == {'time': timedelta(0), 'air_temperature': 275.} + assert new_state == {'time': timedelta(0), 'air_temperature': 276.} @mock.patch.object(MockPrognostic, '__call__') def test_array_no_change_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.zeros((3, 3))}, {}) - state = {'air_temperature': np.ones((3, 3))*273.} + state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() @mock.patch.object(MockPrognostic, '__call__') def test_array_no_change_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*0.}, {}) - state = {'air_temperature': np.ones((3, 3))*273.} + state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() state = new_state diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() state = new_state diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() @mock.patch.object(MockPrognostic, '__call__') def test_array_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*1.}, {}) - state = {'air_temperature': np.ones((3, 3))*273.} + state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*274.).all() @mock.patch.object(MockPrognostic, '__call__') def test_array_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*1.}, {}) - state = {'air_temperature': np.ones((3, 3))*273.} + state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*274.).all() state = new_state diagnostics, new_state = time_stepper.__call__(new_state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*274.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*275.).all() state = new_state diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*275.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*276.).all() @mock.patch.object(MockPrognostic, '__call__') @@ -234,16 +201,16 @@ def test_dataarray_no_change_one_step(self, mock_prognostic_call): {'air_temperature': DataArray(np.zeros((3, 3)), attrs={'units': 'K/s'})}, {}) - state = {'air_temperature': DataArray(np.ones((3, 3))*273., + state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'].values == np.ones((3, 3))*273.).all() assert len(state['air_temperature'].attrs) == 1 assert state['air_temperature'].attrs['units'] == 'K' - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'].values == np.ones((3, 3))*273.).all() assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' @@ -254,32 +221,32 @@ def test_dataarray_no_change_three_steps(self, mock_prognostic_call): {'air_temperature': DataArray(np.zeros((3, 3)), attrs={'units': 'K/s'})}, {}) - state = {'air_temperature': DataArray(np.ones((3, 3))*273., + state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() assert len(state['air_temperature'].attrs) == 1 assert state['air_temperature'].attrs['units'] == 'K' - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() state = new_state assert len(state['air_temperature'].attrs) == 1 assert state['air_temperature'].attrs['units'] == 'K' diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() state = new_state assert len(state['air_temperature'].attrs) == 1 assert state['air_temperature'].attrs['units'] == 'K' diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' @@ -290,16 +257,16 @@ def test_dataarray_one_step(self, mock_prognostic_call): {'air_temperature': DataArray(np.ones((3, 3)), attrs={'units': 'K/s'})}, {}) - state = {'air_temperature': DataArray(np.ones((3, 3))*273., + state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() assert len(state['air_temperature'].attrs) == 1 assert state['air_temperature'].attrs['units'] == 'K' - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*274.).all() assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' @@ -310,32 +277,32 @@ def test_dataarray_three_steps(self, mock_prognostic_call): {'air_temperature': DataArray(np.ones((3, 3)), attrs={'units': 'K/s'})}, {}) - state = {'air_temperature': DataArray(np.ones((3, 3))*273., + state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() assert len(state['air_temperature'].attrs) == 1 assert state['air_temperature'].attrs['units'] == 'K' - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*274.).all() state = new_state assert len(state['air_temperature'].attrs) == 1 assert state['air_temperature'].attrs['units'] == 'K' diagnostics, new_state = time_stepper.__call__(new_state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*274.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*275.).all() state = new_state assert len(state['air_temperature'].attrs) == 1 assert state['air_temperature'].attrs['units'] == 'K' diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*275.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*276.).all() assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' @@ -374,31 +341,31 @@ def timestepper_class(self, *args): def test_array_four_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*1.}, {}) - state = {'air_temperature': np.ones((3, 3))*273.} + state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class([MockPrognostic()]) + time_stepper = self.timestepper_class(MockPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*274.).all() state = new_state diagnostics, new_state = time_stepper.__call__(new_state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*274.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*275.).all() state = new_state diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*275.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*276.).all() state = new_state diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*276.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*277.).all() @@ -406,25 +373,25 @@ def test_array_four_steps(self, mock_prognostic_call): def test_leapfrog_float_two_steps_filtered(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) - state = {'air_temperature': 273.} + state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = Leapfrog([MockPrognostic()], asselin_strength=0.5, alpha=1.) + time_stepper = Leapfrog(MockPrognostic(), asselin_strength=0.5, alpha=1.) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'air_temperature': 273.} - assert new_state == {'air_temperature': 273.} + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} state = new_state mock_prognostic_call.return_value = ({'air_temperature': 2.}, {}) diagnostics, new_state = time_stepper.__call__(state, timestep) # Asselin filter modifies the current state - assert state == {'air_temperature': 274.} - assert new_state == {'air_temperature': 277.} + assert state == {'time': timedelta(0), 'air_temperature': 274.} + assert new_state == {'time': timedelta(0), 'air_temperature': 277.} @mock.patch.object(MockPrognostic, '__call__') def test_leapfrog_requires_same_timestep(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) - state = {'air_temperature': 273.} + state = {'time': timedelta(0), 'air_temperature': 273.} time_stepper = Leapfrog([MockPrognostic()], asselin_strength=0.5) diagnostics, state = time_stepper.__call__(state, timedelta(seconds=1.)) try: @@ -441,8 +408,8 @@ def test_leapfrog_requires_same_timestep(mock_prognostic_call): def test_adams_bashforth_requires_same_timestep(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) - state = {'air_temperature': 273.} - time_stepper = AdamsBashforth([MockPrognostic()]) + state = {'time': timedelta(0), 'air_temperature': 273.} + time_stepper = AdamsBashforth(MockPrognostic()) state = time_stepper.__call__(state, timedelta(seconds=1.)) try: time_stepper.__call__(state, timedelta(seconds=2.)) @@ -460,22 +427,22 @@ def test_leapfrog_array_two_steps_filtered(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*0.}, {}) - state = {'air_temperature': np.ones((3, 3))*273.} + state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = Leapfrog([MockPrognostic()], asselin_strength=0.5, alpha=1.) + time_stepper = Leapfrog(MockPrognostic(), asselin_strength=0.5, alpha=1.) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() state = new_state mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*2.}, {}) diagnostics, new_state = time_stepper.__call__(state, timestep) # Asselin filter modifies the current state - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*274.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*277.).all() @@ -485,22 +452,22 @@ def test_leapfrog_array_two_steps_filtered_williams(mock_prognostic_call): Williams factor of alpha=0.5""" mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*0.}, {}) - state = {'air_temperature': np.ones((3, 3))*273.} + state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = Leapfrog([MockPrognostic()], asselin_strength=0.5, alpha=0.5) + time_stepper = Leapfrog(MockPrognostic(), asselin_strength=0.5, alpha=0.5) diagnostics, new_state = time_stepper.__call__(state, timestep) - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() state = new_state mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*2.}, {}) diagnostics, new_state = time_stepper.__call__(state, timestep) # Asselin filter modifies the current state - assert list(state.keys()) == ['air_temperature'] + assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.5).all() - assert list(new_state.keys()) == ['air_temperature'] + assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*276.5).all() diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py deleted file mode 100644 index 4228a0f..0000000 --- a/tests/test_wrapper.py +++ /dev/null @@ -1,384 +0,0 @@ -from datetime import timedelta, datetime -import unittest -from sympl import ( - Prognostic, Implicit, Diagnostic, UpdateFrequencyWrapper, ScalingWrapper, - TendencyInDiagnosticsWrapper, TimeDifferencingWrapper, DataArray -) -import pytest -from numpy.testing import assert_allclose -from copy import deepcopy - - -class MockPrognostic(Prognostic): - - def __init__(self): - self._num_updates = 0 - - def __call__(self, state): - self._num_updates += 1 - return {}, {'num_updates': self._num_updates} - - -class MockImplicit(Implicit): - - output_properties = { - 'value': { - 'dims': [], - 'units': 'm' - } - } - - diagnostic_properties = { - 'num_updates': { - 'dims': [], - 'units': '' - } - } - - def __init__(self): - self._num_updates = 0 - - def __call__(self, state, timestep): - self._num_updates += 1 - - return ( - {'num_updates': DataArray([self._num_updates], attrs={'units': ''})}, - {'value': DataArray([1], attrs={'units': 'm'})}) - - -class MockImplicitThatExpects(Implicit): - - input_properties = {'expected_field': {}} - output_properties = {'expected_field': {}} - diagnostic_properties = {'expected_field': {}} - - def __init__(self, expected_value): - self._expected_value = expected_value - - def __call__(self, state, timestep): - - input_value = state['expected_field'] - if input_value != self._expected_value: - raise ValueError( - 'Expected {}, but got {}'.format(self._expected_value, input_value)) - - return deepcopy(state), state - - -class MockPrognosticThatExpects(Prognostic): - - input_properties = {'expected_field': {}} - tendency_properties = {'expected_field': {}} - diagnostic_properties = {'expected_field': {}} - - def __init__(self, expected_value): - self._expected_value = expected_value - - def __call__(self, state): - - input_value = state['expected_field'] - if input_value != self._expected_value: - raise ValueError( - 'Expected {}, but got {}'.format(self._expected_value, input_value)) - - return deepcopy(state), state - - -class MockDiagnosticThatExpects(Diagnostic): - - input_properties = {'expected_field': {}} - diagnostic_properties = {'expected_field': {}} - - def __init__(self, expected_value): - self._expected_value = expected_value - - def __call__(self, state): - - input_value = state['expected_field'] - if input_value != self._expected_value: - raise ValueError( - 'Expected {}, but got {}'.format(self._expected_value, input_value)) - - return state - - -class TimeDifferencingTests(unittest.TestCase): - - def setUp(self): - self.implicit = MockImplicit() - self.wrapped = TimeDifferencingWrapper(self.implicit) - self.state = { - 'value': DataArray([0], attrs={'units': 'm'}) - } - - def tearDown(self): - self.component = None - - def testWrapperCallsImplicit(self): - tendencies, diagnostics = self.wrapped(self.state, timedelta(seconds=1)) - assert diagnostics['num_updates'].values[0] == 1 - tendencies, diagnostics = self.wrapped(self.state, timedelta(seconds=1)) - assert diagnostics['num_updates'].values[0] == 2 - assert len(diagnostics.keys()) == 1 - - def testWrapperComputesTendency(self): - tendencies, diagnostics = self.wrapped(self.state, timedelta(seconds=1)) - assert len(tendencies.keys()) == 1 - assert 'value' in tendencies.keys() - assert isinstance(tendencies['value'], DataArray) - assert_allclose(tendencies['value'].to_units('m s^-1').values[0], 1.) - assert_allclose(tendencies['value'].values[0], 1.) - - def testWrapperComputesTendencyWithUnitConversion(self): - state = { - 'value': DataArray([0.011], attrs={'units': 'km'}) - } - tendencies, diagnostics = self.wrapped(state, timedelta(seconds=5)) - assert 'value' in tendencies.keys() - assert isinstance(tendencies['value'], DataArray) - assert_allclose(tendencies['value'].to_units('m s^-1').values[0], -2) - assert_allclose(tendencies['value'].values[0], -2.) - assert_allclose(tendencies['value'].to_units('km s^-1').values[0], -0.002) - - -def test_set_prognostic_update_frequency_calls_initially(): - prognostic = UpdateFrequencyWrapper(MockPrognostic(), timedelta(hours=1)) - state = {'time': timedelta(hours=0)} - tendencies, diagnostics = prognostic(state) - assert len(diagnostics) == 1 - assert diagnostics['num_updates'] == 1 - - -def test_set_prognostic_update_frequency_caches_result(): - prognostic = UpdateFrequencyWrapper(MockPrognostic(), timedelta(hours=1)) - state = {'time': timedelta(hours=0)} - tendencies, diagnostics = prognostic(state) - tendencies, diagnostics = prognostic(state) - assert len(diagnostics) == 1 - assert diagnostics['num_updates'] == 1 - - -def test_set_prognostic_update_frequency_caches_result_with_datetime(): - prognostic = UpdateFrequencyWrapper(MockPrognostic(), timedelta(hours=1)) - state = {'time': datetime(2000, 1, 1)} - tendencies, diagnostics = prognostic(state) - tendencies, diagnostics = prognostic(state) - assert len(diagnostics) == 1 - assert diagnostics['num_updates'] == 1 - - -def test_set_prognostic_update_frequency_updates_result_when_equal(): - prognostic = UpdateFrequencyWrapper(MockPrognostic(), timedelta(hours=1)) - state = {'time': timedelta(hours=0)} - tendencies, diagnostics = prognostic({'time': timedelta(hours=0)}) - tendencies, diagnostics = prognostic({'time': timedelta(hours=1)}) - assert len(diagnostics) == 1 - assert diagnostics['num_updates'] == 2 - - -def test_set_prognostic_update_frequency_updates_result_when_greater(): - prognostic = UpdateFrequencyWrapper(MockPrognostic(), timedelta(hours=1)) - state = {'time': timedelta(hours=0)} - tendencies, diagnostics = prognostic({'time': timedelta(hours=0)}) - tendencies, diagnostics = prognostic({'time': timedelta(hours=2)}) - assert len(diagnostics) == 1 - assert diagnostics['num_updates'] == 2 - - -def test_put_prognostic_tendency_in_diagnostics_no_tendencies(): - class MockPrognostic(Prognostic): - def __call__(self, state): - return {}, {} - - prognostic = TendencyInDiagnosticsWrapper(MockPrognostic(), 'scheme') - tendencies, diagnostics = prognostic({}) - assert len(tendencies) == 0 - assert len(diagnostics) == 0 - - -def test_put_prognostic_tendency_in_diagnostics_one_tendency(): - class MockPrognostic(Prognostic): - tendency_properties = {'quantity': {}} - def __call__(self, state): - return {'quantity': 1.}, {} - - prognostic = TendencyInDiagnosticsWrapper(MockPrognostic(), 'scheme') - tendencies, diagnostics = prognostic({}) - assert 'tendency_of_quantity_due_to_scheme' in prognostic.diagnostics - tendencies, diagnostics = prognostic({}) - assert 'tendency_of_quantity_due_to_scheme' in diagnostics.keys() - assert len(diagnostics) == 1 - assert tendencies['quantity'] == 1. - assert diagnostics['tendency_of_quantity_due_to_scheme'] == 1. - - -def test_scaled_component_wrong_type(): - class WrongType(object): - def __init__(self): - self.a = 1 - - wrong_component = WrongType() - - with pytest.raises(TypeError) as excinfo: - component = ScalingWrapper(wrong_component) - - assert 'either of type Implicit' in str(excinfo.value) - - -def test_scaled_implicit_inputs(): - implicit = ScalingWrapper( - MockImplicitThatExpects(2.0), - input_scale_factors={'expected_field': 0.5}) - - state = {'expected_field': 4.0} - - diagnostics, new_state = implicit(state) - - assert new_state['expected_field'] == 2.0 - assert diagnostics['expected_field'] == 2.0 - - -def test_scaled_implicit_outputs(): - implicit = ScalingWrapper( - MockImplicitThatExpects(4.0), - output_scale_factors={'expected_field': 0.5}) - - state = {'expected_field': 4.0} - - diagnostics, new_state = implicit(state) - - assert new_state['expected_field'] == 2.0 - assert diagnostics['expected_field'] == 4.0 - - -def test_scaled_implicit_diagnostics(): - implicit = ScalingWrapper( - MockImplicitThatExpects(4.0), - diagnostic_scale_factors={'expected_field': 0.5}) - - state = {'expected_field': 4.0} - - diagnostics, new_state = implicit(state) - - assert diagnostics['expected_field'] == 2.0 - assert new_state['expected_field'] == 4.0 - - -def test_scaled_implicit_created_with_wrong_input_field(): - with pytest.raises(ValueError) as excinfo: - implicit = ScalingWrapper( - MockImplicitThatExpects(2.0), - input_scale_factors={'abcd': 0.5}) - - assert 'not a valid input' in str(excinfo.value) - - -def test_scaled_implicit_created_with_wrong_output_field(): - with pytest.raises(ValueError) as excinfo: - implicit = ScalingWrapper( - MockImplicitThatExpects(2.0), - output_scale_factors={'abcd': 0.5}) - - assert 'not a valid output' in str(excinfo.value) - - -def test_scaled_implicit_created_with_wrong_diagnostic_field(): - with pytest.raises(ValueError) as excinfo: - implicit = ScalingWrapper( - MockImplicitThatExpects(2.0), - diagnostic_scale_factors={'abcd': 0.5}) - - assert 'not a valid diagnostic' in str(excinfo.value) - - -def test_scaled_prognostic_inputs(): - prognostic = ScalingWrapper( - MockPrognosticThatExpects(2.0), - input_scale_factors={'expected_field': 0.5}) - - state = {'expected_field': 4.0} - - tendencies, diagnostics = prognostic(state) - - assert tendencies['expected_field'] == 2.0 - assert diagnostics['expected_field'] == 2.0 - - -def test_scaled_prognostic_tendencies(): - prognostic = ScalingWrapper( - MockPrognosticThatExpects(4.0), - tendency_scale_factors={'expected_field': 0.5}) - - state = {'expected_field': 4.0} - - tendencies, diagnostics = prognostic(state) - - assert tendencies['expected_field'] == 2.0 - assert diagnostics['expected_field'] == 4.0 - - -def test_scaled_prognostic_diagnostics(): - prognostic = ScalingWrapper( - MockPrognosticThatExpects(4.0), - diagnostic_scale_factors={'expected_field': 0.5}) - - state = {'expected_field': 4.0} - - tendencies, diagnostics = prognostic(state) - - assert tendencies['expected_field'] == 4.0 - assert diagnostics['expected_field'] == 2.0 - - -def test_scaled_prognostic_with_wrong_tendency_field(): - with pytest.raises(ValueError) as excinfo: - prognostic = ScalingWrapper( - MockPrognosticThatExpects(4.0), - tendency_scale_factors={'abcd': 0.5}) - - assert 'not a valid tendency' in str(excinfo.value) - - -def test_scaled_diagnostic_inputs(): - diagnostic = ScalingWrapper( - MockDiagnosticThatExpects(2.0), - input_scale_factors={'expected_field': 0.5}) - - state = {'expected_field': 4.0} - - diagnostics = diagnostic(state) - - assert diagnostics['expected_field'] == 2.0 - - - -def test_scaled_diagnostic_diagnostics(): - - diagnostic = ScalingWrapper( - MockDiagnosticThatExpects(4.0), - diagnostic_scale_factors = {'expected_field': 0.5}) - - state = {'expected_field': 4.0} - - diagnostics = diagnostic(state) - - assert diagnostics['expected_field'] == 2.0 - - -def test_scaled_component_type_wrongly_modified(): - - diagnostic = ScalingWrapper( - MockDiagnosticThatExpects(4.0), - diagnostic_scale_factors = {'expected_field': 0.5}) - - state = {'expected_field': 4.0} - - diagnostic._component_type = 'abcd' - - with pytest.raises(ValueError) as excinfo: - diagnostics = diagnostic(state) - - assert 'bug in ScalingWrapper' in str(excinfo.value) - -if __name__ == '__main__': - pytest.main([__file__]) From 0f776e6eb75d24bf19e9641dd9d198cf5271a6f2 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 21 Mar 2018 12:23:41 -0700 Subject: [PATCH 11/98] Added prognostic_list note to history --- HISTORY.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 3151928..9868ec7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,10 @@ Latest TendencyInDiagnosticsWrapper is now to be used in Implicit and TimeStepper objects. * Composites now have a component_list attribute which contains the components being composited. +* TimeSteppers now have a prognostic_list attribute which contains the + prognostics used to calculate tendencies. This list should be referenced when + determining inputs and outputs, since TimeSteppers do not currently have + properties dictionaries as attributes. * Added a check for netcdftime having the required objects, to fall back on not using netcdftime when those are missing. This is because most objects are missing in older versions of netcdftime (that come packaged with netCDF4) (closes #23). From 98fe030303acb5b0cbcfae4c5e86ab8a578af6ef Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 21 Mar 2018 13:12:55 -0700 Subject: [PATCH 12/98] Removed print statement --- sympl/_core/composite.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index cd19a33..1ee888f 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -1,7 +1,6 @@ from .exceptions import SharedKeyError from .base_components import Prognostic, Diagnostic, Monitor from .util import update_dict_by_adding_another, ensure_no_shared_keys -import warnings class ComponentComposite(object): @@ -45,7 +44,6 @@ def __init__(self, *args): diagnostic_names = [] for component in self.component_list: diagnostic_names.extend(component.diagnostic_properties.keys()) - print(diagnostic_names) for name in diagnostic_names: if diagnostic_names.count(name) > 1: raise SharedKeyError( From f78795b7b19eb495c8011dd7971b93eef2e8c036 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 22 Mar 2018 12:19:34 -0700 Subject: [PATCH 13/98] Added properties dictionaries to TimeStepper, fixed TimeStepper to pass tests --- HISTORY.rst | 12 +- sympl/__init__.py | 4 +- sympl/_components/basic.py | 1 - sympl/_components/timesteppers.py | 30 ++-- sympl/_core/base_components.py | 32 ++--- sympl/_core/composite.py | 56 ++++++-- sympl/_core/timestepper.py | 114 ++++++++++----- sympl/_core/units.py | 12 ++ sympl/_core/util.py | 115 ++++++++++++++- tests/test_timestepping.py | 231 +++++++++++++++++++++++------- tests/test_util.py | 76 ++++++++-- 11 files changed, 526 insertions(+), 157 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9868ec7..63104ae 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,14 +12,17 @@ Latest * Composites now have a component_list attribute which contains the components being composited. * TimeSteppers now have a prognostic_list attribute which contains the - prognostics used to calculate tendencies. This list should be referenced when - determining inputs and outputs, since TimeSteppers do not currently have - properties dictionaries as attributes. + prognostics used to calculate tendencies. * Added a check for netcdftime having the required objects, to fall back on not using netcdftime when those are missing. This is because most objects are missing in older versions of netcdftime (that come packaged with netCDF4) (closes #23). * TimeSteppers should now be called with individual Prognostics as args, rather than a list of components, and will emit a warning when lists are given. +* TimeSteppers now have input, output, and diagnostic properties as attributes. + These are handled entirely by the base class. +* TimeSteppers now allow you to put tendencies in their diagnostic output. This + is done using first-order time differencing. +* Composites now have properties dictionaries. Breaking changes ~~~~~~~~~~~~~~~~ @@ -32,6 +35,9 @@ Breaking changes __call__ will automatically unwrap DataArrays to numpy arrays to be passed into array_call based on the component's properties dictionaries, and re-wrap to DataArrays when done. +* TimeSteppers should now be written using a _call method rather than __call__. + __call__ wraps _call to provide some base class functionality, like putting + tendencies in diagnostics. * ScalingWrapper, UpdateFrequencyWrapper, and TendencyInDiagnosticsWrapper have been removed. The functionality of these wrappers has been moved to the component base types as methods and initialization options. diff --git a/sympl/__init__.py b/sympl/__init__.py index bcd3419..2f150aa 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -13,7 +13,7 @@ from ._core.constants import ( get_constant, set_constant, set_condensible_name, reset_constants) from ._core.util import ( - combine_dimensions, + combine_array_dimensions, ensure_no_shared_keys, get_numpy_array, jit, restore_dimensions, get_numpy_arrays_with_properties, @@ -35,7 +35,7 @@ InvalidPropertyDictError, DataArray, get_constant, set_constant, set_condensible_name, reset_constants, - TimeDifferencingWrapper, combine_dimensions, + TimeDifferencingWrapper, combine_array_dimensions, ensure_no_shared_keys, get_numpy_array, jit, restore_dimensions, get_numpy_arrays_with_properties, diff --git a/sympl/_components/basic.py b/sympl/_components/basic.py index f55c007..55a426d 100644 --- a/sympl/_components/basic.py +++ b/sympl/_components/basic.py @@ -1,6 +1,5 @@ from .._core.array import DataArray from .._core.base_components import ImplicitPrognostic, Prognostic, Diagnostic -from .._core.array import DataArray from .._core.units import unit_registry as ureg diff --git a/sympl/_components/timesteppers.py b/sympl/_components/timesteppers.py index d209561..2808f8f 100644 --- a/sympl/_components/timesteppers.py +++ b/sympl/_components/timesteppers.py @@ -10,7 +10,7 @@ class SSPRungeKutta(TimeStepper): as proposed by Shu and Osher (1988). """ - def __init__(self, *args, stages=3): + def __init__(self, *args, **kwargs): """ Initialize a strong stability preserving Runge-Kutta time stepper. @@ -18,18 +18,18 @@ def __init__(self, *args, stages=3): ---- *args : Prognostic Objects to call for tendencies when doing time stepping. - stages: int - Number of stages to use. Should be 2 or 3. + stages: int, optional + Number of stages to use. Should be 2 or 3. Default is 3. """ + stages = kwargs.pop('stages', 3) if stages not in (2, 3): raise ValueError( 'stages must be one of 2 or 3, received {}'.format(stages)) self._stages = stages self._euler_stepper = AdamsBashforth(*args, order=1) - super(SSPRungeKutta, self).__init__(*args) + super(SSPRungeKutta, self).__init__(*args, **kwargs) - - def __call__(self, state, timestep): + def _call(self, state, timestep): """ Updates the input state dictionary and returns a new state corresponding to the next timestep. @@ -62,7 +62,6 @@ def _step_3_stages(self, state, timestep): out_state = add(multiply(1./3, state), multiply(2./3, state_2_5)) return diagnostics, out_state - def _step_2_stages(self, state, timestep): assert state is not None diagnostics, state_1 = self._euler_stepper(state, timestep) @@ -75,7 +74,7 @@ def _step_2_stages(self, state, timestep): class AdamsBashforth(TimeStepper): """A TimeStepper using the Adams-Bashforth scheme.""" - def __init__(self, *args, order=3): + def __init__(self, *args, **kwargs): """ Initialize an Adams-Bashforth time stepper. @@ -87,6 +86,7 @@ def __init__(self, *args, order=3): The order of accuracy to use. Must be between 1 and 4. 1 is the same as the Euler method. Default is 3. """ + order = kwargs.pop('order', 3) if isinstance(order, float) and order.is_integer(): order = int(order) if not isinstance(order, int): @@ -96,9 +96,9 @@ def __init__(self, *args, order=3): self._order = order self._timestep = None self._tendencies_list = [] - super(AdamsBashforth, self).__init__(*args) + super(AdamsBashforth, self).__init__(*args, **kwargs) - def __call__(self, state, timestep): + def _call(self, state, timestep): """ Updates the input state dictionary and returns a new state corresponding to the next timestep. @@ -184,7 +184,7 @@ class Leapfrog(TimeStepper): $t_{n+1}$, according to Williams (2009)) closer to the mean of the values at $t_{n-1}$ and $t_{n+1}$.""" - def __init__(self, *args, asselin_strength=0.05, alpha=0.5): + def __init__(self, *args, **kwargs): """ Initialize a Leapfrog time stepper. @@ -210,12 +210,12 @@ def __init__(self, *args, asselin_strength=0.05, alpha=0.5): doi: 10.1175/2009MWR2724.1. """ self._old_state = None - self._asselin_strength = asselin_strength + self._asselin_strength = kwargs.pop('asselin_strength', 0.05) self._timestep = None - self._alpha = alpha - super(Leapfrog, self).__init__(*args) + self._alpha = kwargs.pop('alpha', 0.5) + super(Leapfrog, self).__init__(*args, **kwargs) - def __call__(self, state, timestep): + def _call(self, state, timestep): """ Updates the input state dictionary and returns a new state corresponding to the next timestep. diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index c61fdd9..9c367ed 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -96,8 +96,8 @@ def __repr__(self): self._making_repr = False return return_value - def __init__(self, - input_scale_factors=None, output_scale_factors=None, + def __init__( + self, input_scale_factors=None, output_scale_factors=None, diagnostic_scale_factors=None, tendencies_in_diagnostics=False, update_interval=None, name=None): """ @@ -311,10 +311,6 @@ class Prognostic(object): If not None, the component will only give new output if at least a period of update_interval has passed since the last time new output was given. Otherwise, it would return that cached output. - name : string - A label to be used for this object, for example as would be used for - Y in the name "X_tendency_from_Y". By default the class name in - lowercase is used. """ __metaclass__ = abc.ABCMeta @@ -354,9 +350,9 @@ def __repr__(self): self._making_repr = False return return_value - def __init__(self, - input_scale_factors=None, tendency_scale_factors=None, - diagnostic_scale_factors=None, update_interval=None, name=None): + def __init__( + self, input_scale_factors=None, tendency_scale_factors=None, + diagnostic_scale_factors=None, update_interval=None): """ Initializes the Implicit object. @@ -378,10 +374,6 @@ def __init__(self, If given, the component will only give new output if at least a period of update_interval has passed since the last time new output was given. Otherwise, it would return that cached output. - name : string - A label to be used for this object, for example as would be used for - Y in the name "X_tendency_from_Y". By default the class name in - lowercase is used. """ if input_scale_factors is not None: self.input_scale_factors = input_scale_factors @@ -397,10 +389,6 @@ def __init__(self, self.diagnostic_scale_factors = {} self.update_interval = update_interval self._last_update_time = None - if name is None: - self.name = self.__class__.__name__.lower() - else: - self.name = name def __call__(self, state): """ @@ -549,8 +537,8 @@ def __repr__(self): self._making_repr = False return return_value - def __init__(self, - input_scale_factors=None, tendency_scale_factors=None, + def __init__( + self, input_scale_factors=None, tendency_scale_factors=None, diagnostic_scale_factors=None, update_interval=None, name=None): """ Initializes the Implicit object. @@ -733,8 +721,8 @@ def __repr__(self): self._making_repr = False return return_value - def __init__(self, - input_scale_factors=None, diagnostic_scale_factors=None, + def __init__( + self, input_scale_factors=None, diagnostic_scale_factors=None, update_interval=None): """ Initializes the Implicit object. @@ -859,5 +847,3 @@ def store(self, state): InvalidStateError If state is not a valid input for the Diagnostic instance. """ - - diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 1ee888f..80a5308 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -1,6 +1,8 @@ -from .exceptions import SharedKeyError +from .exceptions import SharedKeyError, InvalidPropertyDictError from .base_components import Prognostic, Diagnostic, Monitor -from .util import update_dict_by_adding_another, ensure_no_shared_keys +from .util import ( + update_dict_by_adding_another, ensure_no_shared_keys, combine_dims, + combine_component_properties) class ComponentComposite(object): @@ -15,6 +17,17 @@ class ComponentComposite(object): component_class = None + @property + def input_properties(self): + return combine_component_properties(self.component_list, 'input_properties') + + @property + def diagnostic_properties(self): + return_dict = {} + for component in self.component_list: + return_dict.update(component.diagnostic_properties) + return return_dict + def __str__(self): return '{}(\n{}\n)'.format( self.__class__, @@ -40,16 +53,17 @@ def __init__(self, *args): if self.component_class is not None: ensure_components_have_class(args, self.component_class) self.component_list = args - if hasattr(self.component_class, 'diagnostic_properties'): - diagnostic_names = [] - for component in self.component_list: - diagnostic_names.extend(component.diagnostic_properties.keys()) - for name in diagnostic_names: - if diagnostic_names.count(name) > 1: - raise SharedKeyError( - 'Two components in a composite should not compute ' - 'the same diagnostic, but multiple passed ' - 'components compute {}'.format(name)) + + def ensure_no_diagnostic_output_overlap(self): + diagnostic_names = [] + for component in self.component_list: + diagnostic_names.extend(component.diagnostic_properties.keys()) + for name in diagnostic_names: + if diagnostic_names.count(name) > 1: + raise SharedKeyError( + 'Two components in a composite should not compute ' + 'the same diagnostic, but multiple passed ' + 'components compute {}'.format(name)) def _combine_attribute(self, attr): return_attr = [] @@ -66,10 +80,22 @@ def ensure_components_have_class(components, component_class): component_class, type(component))) -class PrognosticComposite(ComponentComposite): +class PrognosticComposite(ComponentComposite, Prognostic): component_class = Prognostic + @property + def tendency_properties(self): + return combine_component_properties(self.component_list, 'tendency_properties') + + def __init__(self, *args): + super(PrognosticComposite, self).__init__(*args) + self.ensure_tendency_outputs_are_compatible() + self.ensure_no_diagnostic_output_overlap() + + def ensure_tendency_outputs_are_compatible(self): + self.tendency_properties + def __call__(self, state): """ Gets tendencies and diagnostics from the passed model state. @@ -113,6 +139,10 @@ class DiagnosticComposite(ComponentComposite): component_class = Diagnostic + def __init__(self, *args): + super(DiagnosticComposite, self).__init__(*args) + self.ensure_no_diagnostic_output_overlap() + def __call__(self, state): """ Gets diagnostics from the passed model state. diff --git a/sympl/_core/timestepper.py b/sympl/_core/timestepper.py index 969c5ea..cf03af0 100644 --- a/sympl/_core/timestepper.py +++ b/sympl/_core/timestepper.py @@ -1,6 +1,8 @@ import abc from .composite import PrognosticComposite from .time import timedelta +from .util import combine_component_properties, combine_properties +from .units import clean_units import warnings @@ -37,23 +39,57 @@ class TimeStepper(object): time_unit_timedelta: timedelta A timedelta corresponding to a single time unit as used for time differencing when putting tendencies in diagnostics. + name : string + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". """ __metaclass__ = abc.ABCMeta time_unit_name = 's' time_unit_timedelta = timedelta(seconds=1) + @property + def input_properties(self): + input_properties = combine_component_properties( + self.prognostic_list, 'input_properties') + return combine_properties(input_properties, self.output_properties) + + @property def diagnostic_properties(self): return_value = {} for prognostic in self.prognostic_list: - return_value.update(prognostic.diagnostics) - if self.tendencies_in_diagnostics: - self._insert_tendencies_to_diagnostic_properties(return_value) + return_value.update(prognostic.diagnostic_properties) + if self.tendencies_in_diagnostics: + self._insert_tendencies_to_diagnostic_properties( + return_value, prognostic.tendency_properties, prognostic.name) return return_value - @abc.abstractproperty + def _insert_tendencies_to_diagnostic_properties( + self, diagnostic_properties, tendency_properties, component_name): + for quantity_name, properties in tendency_properties.items(): + tendency_name = self._get_tendency_name(quantity_name, component_name) + if properties['units'] is '': + units = '{}^-1'.format(self.time_unit_name) + else: + units = '{} {}^-1'.format( + properties['units'], self.time_unit_name) + diagnostic_properties[tendency_name] = { + 'units': units, + 'dims': properties['dims'], + } + + @property def output_properties(self): - return {} + output_properties = combine_component_properties( + self.prognostic_list, 'tendency_properties') + for name, properties in output_properties.items(): + properties['units'] += ' {}'.format(self.time_unit_name) + properties['units'] = clean_units(properties['units']) + return output_properties + + @property + def _tendency_properties(self): + return combine_component_properties(self.prognostic_list, 'tendency_properties') def __str__(self): return ( @@ -74,7 +110,7 @@ def __repr__(self): self._making_repr = False return return_value - def __init__(self, *args, tendencies_in_diagnostics=False): + def __init__(self, *args, **kwargs): """ Initialize the TimeStepper. @@ -84,8 +120,14 @@ def __init__(self, *args, tendencies_in_diagnostics=False): Objects to call for tendencies when doing time stepping. tendencies_in_diagnostics : bool, optional A boolean indicating whether this object will put tendencies of - quantities in its diagnostic output. + quantities in its diagnostic output. Default is False. If set to + True, you probably want to give a name also. + name : str + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". By default the class name is used. """ + self.name = kwargs.pop('name', self.__class__.__name__) + tendencies_in_diagnostics = kwargs.pop('tendencies_in_diagnostics', False) if len(args) == 1 and isinstance(args[0], list): warnings.warn( 'TimeSteppers should be given individual Prognostics rather ' @@ -93,30 +135,6 @@ def __init__(self, *args, tendencies_in_diagnostics=False): args = args[0] self._tendencies_in_diagnostics = tendencies_in_diagnostics self.prognostic = PrognosticComposite(*args) - if tendencies_in_diagnostics: - self._insert_tendencies_to_diagnostic_properties() - - def _insert_tendencies_to_diagnostic_properties( - self, diagnostic_properties): - for quantity_name, properties in self.output_properties.items(): - tendency_name = self._get_tendency_name(quantity_name, component_name) - if properties['units'] is '': - units = '{}^-1'.format(self.time_unit_name) - else: - units = '{} {}^-1'.format( - properties['units'], self.time_unit_name) - diagnostic_properties[tendency_name] = { - 'units': units, - 'dims': properties['dims'], - } - - def _insert_tendencies_to_diagnostics( - self, raw_state, raw_new_state, timestep, raw_diagnostics): - for name in self.output_properties.keys(): - tendency_name = self._get_tendency_name(name) - raw_diagnostics[tendency_name] = ( - (raw_new_state[name] - raw_state[name]) / - timestep.total_seconds() * self.time_unit_timedelta.total_seconds()) @property def prognostic_list(self): @@ -126,8 +144,8 @@ def prognostic_list(self): def tendencies_in_diagnostics(self): # value cannot be modified return self._tendencies_in_diagnostics - def _get_tendency_name(self, quantity_name, component_name): - return '{}_tendency_from_{}'.format(quantity_name, component_name) + def _get_tendency_name(self, quantity_name): + return '{}_tendency_from_{}'.format(quantity_name, self.name) def __call__(self, state, timestep): """ @@ -148,6 +166,36 @@ def __call__(self, state, timestep): new_state : dict The model state at the next timestep. """ + diagnostics, new_state = self._call(state, timestep) + if self.tendencies_in_diagnostics: + self._insert_tendencies_to_diagnostics( + state, new_state, timestep, diagnostics) + return diagnostics, new_state + + def _insert_tendencies_to_diagnostics( + self, state, new_state, timestep, diagnostics): + output_properties = self.output_properties + input_properties = self.input_properties + for name in output_properties.keys(): + tendency_name = self._get_tendency_name(name) + if tendency_name in diagnostics.keys(): + raise RuntimeError( + 'A Prognostic has output tendencies as a diagnostic and has' + ' caused a name clash when trying to do so from this ' + 'TimeStepper ({}). You must disable ' + 'tendencies_in_diagnostics for this TimeStepper.'.format( + tendency_name)) + base_units = input_properties[name]['units'] + diagnostics[tendency_name] = ( + (new_state[name].to_units(base_units) - state[name].to_units(base_units)) / + timestep.total_seconds() * self.time_unit_timedelta.total_seconds() + ) + if base_units == '': + diagnostics[tendency_name].attrs['units'] = '{}^-1'.format( + self.time_unit_name) + else: + diagnostics[tendency_name].attrs['units'] = '{} {}^-1'.format( + base_units, self.time_unit_name) @abc.abstractmethod def _call(self, state, timestep): diff --git a/sympl/_core/units.py b/sympl/_core/units.py index c0d46e7..3c4d9fc 100644 --- a/sympl/_core/units.py +++ b/sympl/_core/units.py @@ -19,6 +19,18 @@ def __call__(self, input_string, **kwargs): unit_registry.define('percent = 0.01*count = %') +def units_are_compatible(unit1, unit2): + try: + unit_registry(unit1).to(unit2) + return True + except pint.errors.DimensionalityError: + return False + + +def clean_units(unit_string): + return str(unit_registry(unit_string).to_base_units().units) + + def is_valid_unit(unit_string): """Returns True if the unit string is recognized, and False otherwise.""" unit_string = unit_string.replace( diff --git a/sympl/_core/util.py b/sympl/_core/util.py index 57dfeba..ed27827 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -2,7 +2,7 @@ import numpy as np from six import string_types - +from .units import units_are_compatible from .array import DataArray from .exceptions import ( SharedKeyError, InvalidStateError, InvalidPropertyDictError) @@ -484,7 +484,86 @@ def add_direction_names(x=None, y=None, z=None): dim_names[key].extend(value) -def combine_dimensions(arrays, out_dims): +def get_direction_name(dims, direction): + for name in dims: + if name in dim_names[direction] or name == direction: + return name + return None + + +def combine_dims(dims1, dims2): + """ + Takes in two dims specifications and returns a single specification that + satisfies both, if possible. Raises an InvalidPropertyDictError if not. + + Parameters + ---------- + dims1 : iterable of str + dims2 : iterable of str + + Returns + ------- + dims : iterable of str + + Raises + ------ + InvalidPropertyDictError + If the two dims specifications cannot be combined + """ + if dims1 == dims2: + return dims1 + dims_out = [] + for direction in dim_names.keys(): + dir1 = get_direction_name(dims1, direction) + dir2 = get_direction_name(dims2, direction) + if dir1 is None and dir2 is None: + pass + elif dir1 is None and dir2 is not None: + dims_out.append(dir2) + elif dir1 is not None and dir2 is None: + dims_out.append(dir1) + elif dir1 == direction: + dims_out.append(dir2) + elif dir2 == direction: + dims_out.append(dir1) + elif dir1 == dir2: + dims_out.append(dir1) + else: + raise InvalidPropertyDictError( + 'Two dims {} and {} exist for direction {}'.format( + dir1, dir2, direction)) + dims1 = set(dims1).difference(dims_out) + dims2 = set(dims2).difference(dims_out) + dims1_wildcard = '*' in dims1 + dims1.discard('*') + dims1 = dims1.difference(dim_names.keys()) + dims2_wildcard = '*' in dims2 + dims2.discard('*') + dims2 = dims2.difference(dim_names.keys()) + unmatched_dims = set(dims1).union(dims2).difference(dims_out) + shared_dims = set(dims2).intersection(dims2) + if dims1_wildcard and dims2_wildcard: + dims_out.insert(0, '*') # either dim can match anything + dims_out.extend(unmatched_dims) + elif not dims1_wildcard and not dims2_wildcard: + if shared_dims != set(dims1) or shared_dims != set(dims2): + raise InvalidPropertyDictError( + 'dims {} and {} are incompatible'.format(dims1, dims2)) + dims_out.extend(unmatched_dims) + elif dims1_wildcard: + if shared_dims != set(dims2): + raise InvalidPropertyDictError( + 'dims {} and {} are incompatible'.format(dims1, dims2)) + dims_out.extend(unmatched_dims) + elif dims2_wildcard: + if shared_dims != set(dims1): + raise InvalidPropertyDictError( + 'dims {} and {} are incompatible'.format(dims1, dims2)) + dims_out.extend(unmatched_dims) + return dims_out + + +def combine_array_dimensions(arrays, out_dims): """ Returns a tuple of dimension names corresponding to dimension names from the DataArray objects given by *args when present. @@ -664,3 +743,35 @@ def get_final_shape(data_array, out_dims, direction_to_names): np.product([len(data_array.coords[name]) for name in direction_to_names[direction]])) return final_shape + + +def combine_component_properties(component_list, property_name): + args = [] + for component in component_list: + args.append(getattr(component, property_name)) + return combine_properties(*args) + + +def combine_properties(*args): + return_dict = {} + for property_dict in args: + for name, properties in property_dict.items(): + if name not in return_dict: + return_dict[name] = {} + return_dict[name].update(properties) + elif not units_are_compatible( + properties['units'], return_dict[name]['units']): + raise InvalidPropertyDictError( + 'Cannot combine components with incompatible units ' + '{} and {} for quantity {}'.format( + return_dict[name]['units'], + properties['units'], name)) + else: + try: + dims = combine_dims(return_dict[name]['dims'], properties['dims']) + return_dict[name]['dims'] = dims + except InvalidPropertyDictError as err: + raise InvalidPropertyDictError( + 'Incompatibility between dims of quantity {}: {}'.format( + name, err.args[0])) + return return_dict diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index eafaccc..8de3d24 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -1,7 +1,9 @@ import pytest import mock from sympl import ( - Prognostic, Leapfrog, AdamsBashforth, DataArray, SSPRungeKutta, timedelta) + Prognostic, Leapfrog, AdamsBashforth, DataArray, SSPRungeKutta, timedelta, + InvalidPropertyDictError) +from sympl._core.units import units_are_compatible import numpy as np @@ -10,7 +12,7 @@ def same_list(list1, list2): [item in list2 for item in list1] + [item in list1 for item in list2])) -class MockPrognostic(Prognostic): +class MockEmptyPrognostic(Prognostic): input_properties = {} tendency_properties = {} @@ -20,53 +22,77 @@ def array_call(self, state): return {}, {} +class MockPrognostic(Prognostic): + + input_properties = None + diagnostic_properties = None + tendency_properties = None + + def __init__( + self, input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, **kwargs): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self.tendency_properties = tendency_properties + self._diagnostic_output = diagnostic_output + self._tendency_output = tendency_output + self.times_called = 0 + self.state_given = None + super(MockPrognostic, self).__init__(**kwargs) + + def array_call(self, state): + self.times_called += 1 + self.state_given = state + return self._tendency_output, self._diagnostic_output + + class TimesteppingBase(object): timestepper_class = None def test_unused_quantities_carried_over(self): state = {'time': timedelta(0), 'air_temperature': 273.} - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) timestep = timedelta(seconds=1.) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} def test_timestepper_reveals_prognostics(self): - prog1 = MockPrognostic() + prog1 = MockEmptyPrognostic() prog1.input_properties = {'input1': {}} time_stepper = self.timestepper_class(prog1) assert same_list(time_stepper.prognostic_list, (prog1,)) - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_float_no_change_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} assert diagnostics == {} - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_float_no_change_one_step_diagnostic(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': 0.}, {'foo': 'bar'}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} assert diagnostics == {'foo': 'bar'} - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_float_no_change_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} @@ -79,34 +105,34 @@ def test_float_no_change_three_steps(self, mock_prognostic_call): assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_float_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 1.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 274.} - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_float_one_step_with_units(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'eastward_wind': DataArray(0.02, attrs={'units': 'km/s^2'})}, {}) state = {'time': timedelta(0), 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} assert same_list(new_state.keys(), ['time', 'eastward_wind']) assert np.allclose(new_state['eastward_wind'].values, 21.) assert new_state['eastward_wind'].attrs['units'] == 'm/s' - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_float_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 1.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 274.} @@ -119,26 +145,26 @@ def test_float_three_steps(self, mock_prognostic_call): assert state == {'time': timedelta(0), 'air_temperature': 275.} assert new_state == {'time': timedelta(0), 'air_temperature': 276.} - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_array_no_change_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.zeros((3, 3))}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_array_no_change_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*0.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -157,26 +183,26 @@ def test_array_no_change_three_steps(self, mock_prognostic_call): assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_array_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*1.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*274.).all() - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_array_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*1.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -195,7 +221,7 @@ def test_array_three_steps(self, mock_prognostic_call): assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*276.).all() - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_dataarray_no_change_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': @@ -204,7 +230,7 @@ def test_dataarray_no_change_one_step(self, mock_prognostic_call): state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'].values == np.ones((3, 3))*273.).all() @@ -215,7 +241,7 @@ def test_dataarray_no_change_one_step(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_dataarray_no_change_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': @@ -224,7 +250,7 @@ def test_dataarray_no_change_three_steps(self, mock_prognostic_call): state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -251,7 +277,7 @@ def test_dataarray_no_change_three_steps(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_dataarray_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': @@ -260,7 +286,7 @@ def test_dataarray_one_step(self, mock_prognostic_call): state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -271,7 +297,7 @@ def test_dataarray_one_step(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' - @mock.patch.object(MockPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_dataarray_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': @@ -280,7 +306,7 @@ def test_dataarray_three_steps(self, mock_prognostic_call): state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -307,43 +333,140 @@ def test_dataarray_three_steps(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' + def test_tendencies_in_diagnostics_no_tendency(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + prognostic = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class( + prognostic, tendencies_in_diagnostics=True) + state = {'time': timedelta(0)} + diagnostics, _ = stepper(state, timedelta(seconds=5)) + assert diagnostics == {} + + def test_tendencies_in_diagnostics_one_tendency(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + } + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class( + prognostic, tendencies_in_diagnostics=True) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10])*10., + dims=['dim1'], + attrs={'units': 'm'} + ), + } + diagnostics, _ = stepper(state, timedelta(seconds=5)) + tendency_name = 'output1_tendency_from_{}'.format(stepper.__class__.__name__) + assert tendency_name in diagnostics.keys() + assert len( + diagnostics[tendency_name].dims) == 1 + assert 'dim1' in diagnostics[tendency_name].dims + assert units_are_compatible(diagnostics[tendency_name].attrs['units'], 'm s^-1') + assert np.allclose(diagnostics[tendency_name].values, 2.) + + def test_tendencies_in_diagnostics_one_tendency_with_component_name(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + } + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class( + prognostic, tendencies_in_diagnostics=True, name='component') + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10])*10., + dims=['dim1'], + attrs={'units': 'm'} + ), + } + diagnostics, _ = stepper(state, timedelta(seconds=5)) + assert 'output1_tendency_from_component' in diagnostics.keys() + assert len( + diagnostics['output1_tendency_from_component'].dims) == 1 + assert 'dim1' in diagnostics['output1_tendency_from_component'].dims + assert units_are_compatible(diagnostics['output1_tendency_from_component'].attrs['units'], 'm s^-1') + assert np.allclose(diagnostics['output1_tendency_from_component'].values, 2.) + class TestSSPRungeKuttaTwoStep(TimesteppingBase): - def timestepper_class(self, *args): - return SSPRungeKutta(*args, stages=2) + def timestepper_class(self, *args, **kwargs): + kwargs['stages'] = 2 + return SSPRungeKutta(*args, **kwargs) class TestSSPRungeKuttaThreeStep(TimesteppingBase): - def timestepper_class(self, *args): - return SSPRungeKutta(*args, stages=3) + + def timestepper_class(self, *args, **kwargs): + kwargs['stages'] = 3 + return SSPRungeKutta(*args, **kwargs) class TestLeapfrog(TimesteppingBase): + timestepper_class = Leapfrog class TestAdamsBashforthSecondOrder(TimesteppingBase): - def timestepper_class(self, *args): - return AdamsBashforth(*args, order=2) + + def timestepper_class(self, *args, **kwargs): + kwargs['order'] = 2 + return AdamsBashforth(*args, **kwargs) class TestAdamsBashforthThirdOrder(TimesteppingBase): - def timestepper_class(self, *args): - return AdamsBashforth(*args, order=3) + + def timestepper_class(self, *args, **kwargs): + kwargs['order'] = 3 + return AdamsBashforth(*args, **kwargs) class TestAdamsBashforthFourthOrder(TimesteppingBase): - def timestepper_class(self, *args): - return AdamsBashforth(*args, order=4) - @mock.patch.object(MockPrognostic, '__call__') + def timestepper_class(self, *args, **kwargs): + kwargs['order'] = 4 + return AdamsBashforth(*args, **kwargs) + + @mock.patch.object(MockEmptyPrognostic, '__call__') def test_array_four_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*1.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -369,13 +492,13 @@ def test_array_four_steps(self, mock_prognostic_call): assert (new_state['air_temperature'] == np.ones((3, 3))*277.).all() -@mock.patch.object(MockPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognostic, '__call__') def test_leapfrog_float_two_steps_filtered(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = Leapfrog(MockPrognostic(), asselin_strength=0.5, alpha=1.) + time_stepper = Leapfrog(MockEmptyPrognostic(), asselin_strength=0.5, alpha=1.) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} @@ -387,12 +510,12 @@ def test_leapfrog_float_two_steps_filtered(mock_prognostic_call): assert new_state == {'time': timedelta(0), 'air_temperature': 277.} -@mock.patch.object(MockPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognostic, '__call__') def test_leapfrog_requires_same_timestep(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} - time_stepper = Leapfrog([MockPrognostic()], asselin_strength=0.5) + time_stepper = Leapfrog([MockEmptyPrognostic()], asselin_strength=0.5) diagnostics, state = time_stepper.__call__(state, timedelta(seconds=1.)) try: time_stepper.__call__(state, timedelta(seconds=2.)) @@ -404,12 +527,12 @@ def test_leapfrog_requires_same_timestep(mock_prognostic_call): raise AssertionError('Leapfrog must require timestep to be constant') -@mock.patch.object(MockPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognostic, '__call__') def test_adams_bashforth_requires_same_timestep(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} - time_stepper = AdamsBashforth(MockPrognostic()) + time_stepper = AdamsBashforth(MockEmptyPrognostic()) state = time_stepper.__call__(state, timedelta(seconds=1.)) try: time_stepper.__call__(state, timedelta(seconds=2.)) @@ -422,14 +545,14 @@ def test_adams_bashforth_requires_same_timestep(mock_prognostic_call): 'AdamsBashforth must require timestep to be constant') -@mock.patch.object(MockPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognostic, '__call__') def test_leapfrog_array_two_steps_filtered(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*0.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = Leapfrog(MockPrognostic(), asselin_strength=0.5, alpha=1.) + time_stepper = Leapfrog(MockEmptyPrognostic(), asselin_strength=0.5, alpha=1.) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -446,7 +569,7 @@ def test_leapfrog_array_two_steps_filtered(mock_prognostic_call): assert (new_state['air_temperature'] == np.ones((3, 3))*277.).all() -@mock.patch.object(MockPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognostic, '__call__') def test_leapfrog_array_two_steps_filtered_williams(mock_prognostic_call): """Test that the Asselin filter is being correctly applied with a Williams factor of alpha=0.5""" @@ -454,7 +577,7 @@ def test_leapfrog_array_two_steps_filtered_williams(mock_prognostic_call): {'air_temperature': np.ones((3, 3))*0.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = Leapfrog(MockPrognostic(), asselin_strength=0.5, alpha=0.5) + time_stepper = Leapfrog(MockEmptyPrognostic(), asselin_strength=0.5, alpha=0.5) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() diff --git a/tests/test_util.py b/tests/test_util.py index cdf84da..dc81e09 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,12 +1,11 @@ import unittest -from copy import deepcopy - import numpy as np import pytest from sympl import ( Prognostic, ensure_no_shared_keys, SharedKeyError, DataArray, - combine_dimensions, set_direction_names, Implicit, Diagnostic) -from sympl._core.util import update_dict_by_adding_another + combine_array_dimensions, set_direction_names, Implicit, Diagnostic, + InvalidPropertyDictError) +from sympl._core.util import update_dict_by_adding_another, combine_dims def same_list(list1, list2): @@ -78,7 +77,7 @@ def test_ensure_no_shared_keys_with_shared_keys(): 'No exception raised but expected SharedKeyError.') -class CombineDimensionsTests(unittest.TestCase): +class CombineArrayDimensionsTests(unittest.TestCase): def setUp(self): self.array_1d = DataArray(np.zeros((2,)), dims=['lon']) @@ -92,29 +91,29 @@ def tearDown(self): set_direction_names(x=[], y=[], z=[]) def test_combine_dimensions_2d_and_3d(self): - dims = combine_dimensions( + dims = combine_array_dimensions( [self.array_2d, self.array_3d], out_dims=('x', 'y', 'z')) assert same_list(dims, ['lon', 'lat', 'interface_levels']) def test_combine_dimensions_2d_and_3d_z_y_x(self): - dims = combine_dimensions( + dims = combine_array_dimensions( [self.array_2d, self.array_3d], out_dims=('z', 'y', 'x')) assert same_list(dims, ['interface_levels', 'lat', 'lon']) def combine_dimensions_1d_shared(self): - dims = combine_dimensions( + dims = combine_array_dimensions( [self.array_1d, self.array_1d], out_dims=['x']) assert same_list(dims, ['lon']) def combine_dimensions_1d_not_shared(self): array_1d_x = DataArray(np.zeros((2,)), dims=['lon']) array_1d_y = DataArray(np.zeros((2,)), dims=['lat']) - dims = combine_dimensions([array_1d_x, array_1d_y], out_dims=['x', 'y']) + dims = combine_array_dimensions([array_1d_x, array_1d_y], out_dims=['x', 'y']) assert same_list(dims, ['lon', 'lat']) def combine_dimensions_1d_wrong_direction(self): try: - combine_dimensions( + combine_array_dimensions( [self.array_1d, self.array_1d], out_dims=['z']) except ValueError: pass @@ -125,7 +124,7 @@ def combine_dimensions_1d_wrong_direction(self): def combine_dimensions_1d_and_2d_extra_direction(self): try: - combine_dimensions( + combine_array_dimensions( [self.array_1d, self.array_2d], out_dims=['y']) except ValueError: pass @@ -135,5 +134,60 @@ def combine_dimensions_1d_and_2d_extra_direction(self): raise AssertionError('No exception raised but expected ValueError.') +class CombineDimsTests(unittest.TestCase): + + def setUp(self): + set_direction_names( + x=['lon'], y=['lat'], z=['mid_levels', 'interface_levels']) + + def tearDown(self): + set_direction_names(x=[], y=[], z=[]) + + def test_same_dims(self): + assert combine_dims(['dim1'], ['dim1']) == ['dim1'] + + def test_one_wildcard(self): + assert combine_dims(['*'], ['dim1']) == ['dim1'] + + def test_both_wildcard(self): + assert combine_dims(['*'], ['*']) == ['*'] + + def test_both_wildcard_2d(self): + assert set(combine_dims(['*', 'dim1'], ['*', 'dim1'])) == {'*', 'dim1'} + + def test_one_wildcard_2d(self): + assert set(combine_dims(['*', 'dim2'], ['dim1', 'dim2'])) == {'dim1', 'dim2'} + + def test_different_dims(self): + with self.assertRaises(InvalidPropertyDictError): + combine_dims(['dim1'], ['dim2']) + + def test_swapped_dims(self): + result = set(combine_dims(['dim1', 'dim2'], ['dim2', 'dim1'])) + assert result == {'dim1', 'dim2'} + + def test_swapped_dims_with_wildcard(self): + result = combine_dims(['*', 'dim1', 'dim2'], ['*', 'dim2', 'dim1']) + assert set(result) == {'*', 'dim1', 'dim2'} + + def test_one_directional(self): + set_direction_names(x=['lon']) + assert combine_dims(['lon'], ['lon']) == ['lon'] + + def test_overlapping_directional(self): + set_direction_names(x=['lon_mid', 'lon_edge']) + with self.assertRaises(InvalidPropertyDictError): + combine_dims(['lon_mid'], ['lon_edge']) + + def test_combine_directional_wildcard(self): + set_direction_names(x=['lon']) + assert combine_dims(['x'], ['lon']) == ['lon'] + + def test_combine_multiple_directional_wildcard(self): + set_direction_names(x=['lon']) + result = combine_dims(['*', 'x', 'y'], ['*', 'lon']) + assert set(result) == {'*', 'lon', 'y'} + + if __name__ == '__main__': pytest.main([__file__]) From 18cb7cac855f11273a90ed2332245f83ebf434a4 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 22 Mar 2018 12:34:23 -0700 Subject: [PATCH 14/98] flake8 fixes --- sympl/__init__.py | 2 +- sympl/_core/composite.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/sympl/__init__.py b/sympl/__init__.py index 2f150aa..e868424 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -30,7 +30,7 @@ __all__ = ( Prognostic, Diagnostic, Implicit, Monitor, PrognosticComposite, DiagnosticComposite, MonitorComposite, ImplicitPrognostic, - TimeStepper, Leapfrog, AdamsBashforth, + TimeStepper, Leapfrog, AdamsBashforth, SSPRungeKutta, InvalidStateError, SharedKeyError, DependencyError, InvalidPropertyDictError, DataArray, diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 80a5308..c6c6d20 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -1,7 +1,7 @@ -from .exceptions import SharedKeyError, InvalidPropertyDictError +from .exceptions import SharedKeyError, from .base_components import Prognostic, Diagnostic, Monitor from .util import ( - update_dict_by_adding_another, ensure_no_shared_keys, combine_dims, + update_dict_by_adding_another, ensure_no_shared_keys, combine_component_properties) @@ -134,8 +134,11 @@ def __call__(self, state): return_diagnostics.update(diagnostics) return return_tendencies, return_diagnostics + def array_call(self, state): + raise NotImplementedError() -class DiagnosticComposite(ComponentComposite): + +class DiagnosticComposite(ComponentComposite, Diagnostic): component_class = Diagnostic @@ -177,6 +180,8 @@ def __call__(self, state): return_diagnostics.update(diagnostics) return return_diagnostics + def array_call(self, state): + raise NotImplementedError() class MonitorComposite(ComponentComposite): From dc7ab57a80b1125893342fcb30ce4aa41ae12f55 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 22 Mar 2018 14:37:58 -0700 Subject: [PATCH 15/98] Fixed to pass tests after merge --- sympl/__init__.py | 2 +- sympl/_core/composite.py | 2 +- tests/test_util.py | 14 +------------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/sympl/__init__.py b/sympl/__init__.py index a0704bf..dbb3663 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -37,7 +37,7 @@ InvalidPropertyDictError, DataArray, get_constant, set_constant, set_condensible_name, reset_constants, - get_constants_string,TimeDifferencingWrapper, combine_dimensions, + get_constants_string,TimeDifferencingWrapper, ensure_no_shared_keys, get_numpy_array, jit, restore_dimensions, get_numpy_arrays_with_properties, diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index c6c6d20..ba50e8a 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -1,4 +1,4 @@ -from .exceptions import SharedKeyError, +from .exceptions import SharedKeyError from .base_components import Prognostic, Diagnostic, Monitor from .util import ( update_dict_by_adding_another, ensure_no_shared_keys, diff --git a/tests/test_util.py b/tests/test_util.py index f36af2a..f334e10 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -88,8 +88,7 @@ def test_get_component_aliases_with_no_args(): def test_get_component_aliases_with_single_component_arg(): - components = [MockPrognostic(), MockImplicit(), MockDiagnostic(), - TendencyInDiagnosticsWrapper(DummyPrognostic(), 'dummy')] + components = [MockPrognostic(), MockImplicit(), MockDiagnostic()] for c, comp in enumerate(components): aliases = get_component_aliases(comp) assert type(aliases) == dict @@ -101,17 +100,6 @@ def test_get_component_aliases_with_single_component_arg(): assert len(aliases.keys()) == 0 -def test_get_component_aliases_with_two_component_args(): - components = [MockDiagnostic(), MockImplicit(), MockDiagnostic(), - TendencyInDiagnosticsWrapper(DummyPrognostic(), 'dummy')] - for comp in components[:3]: - aliases = get_component_aliases(comp, components[-1]) - assert type(aliases) == dict - assert len(aliases.keys()) == 2 - for k in ['T', 'P']: - assert k in list(aliases.values()) - - class DummyProg1(Prognostic): input_properties = {'temperature': {'alias': 'T'}} tendency_properties = {'temperature': {'alias': 'TEMP'}} From ed6326f986a5aca4bc07461fe2ade66fb53433d0 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 22 Mar 2018 23:16:03 -0700 Subject: [PATCH 16/98] Added properties tests for TimeStepper --- sympl/_core/timestepper.py | 25 +++--- tests/test_timestepping.py | 169 +++++++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+), 11 deletions(-) diff --git a/sympl/_core/timestepper.py b/sympl/_core/timestepper.py index cf03af0..83ff010 100644 --- a/sympl/_core/timestepper.py +++ b/sympl/_core/timestepper.py @@ -3,6 +3,7 @@ from .time import timedelta from .util import combine_component_properties, combine_properties from .units import clean_units +from .state import copy_untouched_quantities import warnings @@ -59,22 +60,19 @@ def diagnostic_properties(self): return_value = {} for prognostic in self.prognostic_list: return_value.update(prognostic.diagnostic_properties) - if self.tendencies_in_diagnostics: - self._insert_tendencies_to_diagnostic_properties( - return_value, prognostic.tendency_properties, prognostic.name) + if self.tendencies_in_diagnostics: + tendency_properties = combine_component_properties( + self.prognostic_list, 'tendency_properties') + self._insert_tendencies_to_diagnostic_properties( + return_value, tendency_properties) return return_value def _insert_tendencies_to_diagnostic_properties( - self, diagnostic_properties, tendency_properties, component_name): + self, diagnostic_properties, tendency_properties): for quantity_name, properties in tendency_properties.items(): - tendency_name = self._get_tendency_name(quantity_name, component_name) - if properties['units'] is '': - units = '{}^-1'.format(self.time_unit_name) - else: - units = '{} {}^-1'.format( - properties['units'], self.time_unit_name) + tendency_name = self._get_tendency_name(quantity_name) diagnostic_properties[tendency_name] = { - 'units': units, + 'units': properties['units'], 'dims': properties['dims'], } @@ -128,6 +126,10 @@ def __init__(self, *args, **kwargs): """ self.name = kwargs.pop('name', self.__class__.__name__) tendencies_in_diagnostics = kwargs.pop('tendencies_in_diagnostics', False) + if len(kwargs) > 0: + raise TypeError( + "TimeStepper.__init__ got an unexpected keyword argument '{}'".format( + kwargs.popitem()[0])) if len(args) == 1 and isinstance(args[0], list): warnings.warn( 'TimeSteppers should be given individual Prognostics rather ' @@ -167,6 +169,7 @@ def __call__(self, state, timestep): The model state at the next timestep. """ diagnostics, new_state = self._call(state, timestep) + copy_untouched_quantities(state, new_state) if self.tendencies_in_diagnostics: self._insert_tendencies_to_diagnostics( state, new_state, timestep, diagnostics) diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index 8de3d24..73ea226 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -420,6 +420,175 @@ def test_tendencies_in_diagnostics_one_tendency_with_component_name(self): assert units_are_compatible(diagnostics['output1_tendency_from_component'].attrs['units'], 'm s^-1') assert np.allclose(diagnostics['output1_tendency_from_component'].values, 2.) + def test_copies_untouched_quantities(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + }, + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class( + prognostic, tendencies_in_diagnostics=True, name='component') + untouched_quantity = DataArray( + np.ones([10])*10., + dims=['dim1'], + attrs={'units': 'J'} + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10])*10., + dims=['dim1'], + attrs={'units': 'm'} + ), + 'input1': untouched_quantity, + } + _, new_state = stepper(state, timedelta(seconds=5)) + assert 'input1' in new_state.keys() + print(new_state['input1'].values.data, untouched_quantity.values.data) + assert new_state['input1'].dims == untouched_quantity.dims + assert np.allclose(new_state['input1'].values, 10.) + assert new_state['input1'].attrs['units'] == 'J' + + def test_stepper_requires_input_for_stepped_quantity(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + }, + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class(prognostic) + assert 'output1' in stepper.input_properties.keys() + assert stepper.input_properties['output1']['dims'] == ['dim1'] + assert units_are_compatible(stepper.input_properties['output1']['units'], 'm') + + def test_stepper_outputs_stepped_quantity(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + }, + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class(prognostic) + assert 'output1' in stepper.output_properties.keys() + assert stepper.output_properties['output1']['dims'] == ['dim1'] + assert units_are_compatible(stepper.output_properties['output1']['units'], 'm') + + def test_stepper_requires_input_for_input_quantity(self): + input_properties = { + 'input1': { + 'dims': ['dim1', 'dim2'], + 'units': 's', + } + } + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + }, + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class(prognostic) + assert 'input1' in stepper.input_properties.keys() + assert stepper.input_properties['input1']['dims'] == ['dim1', 'dim2'] + assert units_are_compatible(stepper.input_properties['input1']['units'], 's') + assert len(stepper.diagnostic_properties) == 0 + + def test_stepper_gives_diagnostic_quantity(self): + input_properties = {} + diagnostic_properties = { + 'diag1': { + 'dims': ['dim2'], + 'units': '', + } + } + tendency_properties = { + } + diagnostic_output = {} + tendency_output = { + } + prognostic = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class( + prognostic, tendencies_in_diagnostics=True, name='component') + assert 'diag1' in stepper.diagnostic_properties.keys() + assert stepper.diagnostic_properties['diag1']['dims'] == ['dim2'] + assert units_are_compatible( + stepper.diagnostic_properties['diag1']['units'], '') + assert len(stepper.input_properties) == 0 + assert len(stepper.output_properties) == 0 + + def test_stepper_gives_diagnostic_tendency_quantity(self): + input_properties = { + 'input1': { + 'dims': ['dim1', 'dim2'], + 'units': 's', + } + } + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + }, + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class( + prognostic, tendencies_in_diagnostics=True) + tendency_name = 'output1_tendency_from_{}'.format(stepper.__class__.__name__) + assert tendency_name in stepper.diagnostic_properties.keys() + assert len(stepper.diagnostic_properties) == 1 + assert stepper.diagnostic_properties[tendency_name]['dims'] == ['dim1'] + assert units_are_compatible(stepper.input_properties['output1']['units'], 'm') + assert units_are_compatible(stepper.diagnostic_properties[tendency_name]['units'], 'm/s') + class TestSSPRungeKuttaTwoStep(TimesteppingBase): From 6b185ca5f6bac09942cc484194c066f849e597bf Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 22 Mar 2018 23:37:19 -0700 Subject: [PATCH 17/98] Removed directional wildcards 'x', 'y', 'z' --- HISTORY.rst | 4 + sympl/__init__.py | 4 +- sympl/_core/util.py | 130 ++------------- tests/test_get_restore_numpy_array.py | 224 +------------------------- tests/test_match_dims_like.py | 74 +-------- tests/test_util.py | 84 +--------- 6 files changed, 21 insertions(+), 499 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 581c856..51dada0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -44,6 +44,10 @@ Breaking changes * 'time' now must be present in the model state dictionary. This is strictly required for calls to Diagnostic, Prognostic, ImplicitPrognostic, and Implicit components, and may be strictly required in other ways in the future +* Removed everything to do with directional wildcards. Currently '*' is the + only wildcard dimension. 'x', 'y', and 'z' refer to their own names only. +* Removed the combine_dimensions function, which wasn't used anywhere and no + longer has much purpose without directional wildcards v0.3.2 ------ diff --git a/sympl/__init__.py b/sympl/__init__.py index dbb3663..bad30f3 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -14,12 +14,10 @@ get_constant, set_constant, set_condensible_name, reset_constants, get_constants_string) from ._core.util import ( - combine_array_dimensions, ensure_no_shared_keys, get_numpy_array, jit, restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, - set_direction_names, add_direction_names, get_component_aliases) from ._core.testing import ComponentTestBase from ._components import ( @@ -42,7 +40,7 @@ get_numpy_array, jit, restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, - set_direction_names, add_direction_names, get_component_aliases, + get_component_aliases, ComponentTestBase, PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, diff --git a/sympl/_core/util.py b/sympl/_core/util.py index f4e85cd..24005ef 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -1,7 +1,5 @@ from datetime import datetime - import numpy as np -from six import string_types from .units import units_are_compatible from .array import DataArray from .exceptions import ( @@ -17,8 +15,6 @@ def jit(signature_or_function=None, **kwargs): else: return signature_or_function -dim_names = {'x': ['x'], 'y': ['y'], 'z': ['z']} - # internal exceptions used only within this module @@ -210,8 +206,8 @@ def get_numpy_array( which is the flattened collection of all dimensions not explicitly listed in out_dims, including any dimensions with unknown direction. return_wildcard_matches : bool, optional - If True, will additionally return a dictionary whose keys are direciton - wildcards ('x', 'y', 'z', or '*') and values are lists of matched + If True, will additionally return a dictionary whose keys are direction + wildcards (currently only '*') and values are lists of matched dimensions in the order they appear. require_wildcard_matches : dict, optional A dictionary mapping wildcards to matches. If the wildcard is used in @@ -231,13 +227,15 @@ def get_numpy_array( in data_array, or data_array's dimensions are invalid in some way. """ + # This function was written when we had directional wildcards, and could + # be re-written to be simpler now that we do not. if (len(data_array.values.shape) == 0) and (len(out_dims) == 0): direction_to_names = {} # required in case we need wildcard_matches return_array = data_array.values # special case, 0-dimensional scalar array else: - current_dim_names = dim_names.copy() + current_dim_names = {} for dim in out_dims: - if dim not in ('x', 'y', 'z', '*'): + if dim != '*': current_dim_names[dim] = [dim] direction_to_names = get_input_array_dim_names( data_array, out_dims, current_dim_names) @@ -266,7 +264,7 @@ def get_numpy_array( if return_wildcard_matches: wildcard_matches = { key: value for key, value in direction_to_names.items() - if key in ('x', 'y', 'z', '*')} + if key == '*'} return return_array, wildcard_matches else: return return_array @@ -421,9 +419,9 @@ def restore_dimensions(array, from_dims, result_like, result_attrs=None): :py:function:~sympl.get_numpy_array: : Retrieves a numpy array with desired dimensions from a given DataArray. """ - current_dim_names = dim_names.copy() + current_dim_names = {} for dim in from_dims: - if dim not in ('x', 'y', 'z', '*'): + if dim != '*': current_dim_names[dim] = [dim] direction_to_names = get_input_array_dim_names( result_like, from_dims, current_dim_names) @@ -460,37 +458,6 @@ def same_list(list1, list2): [item in list2 for item in list1] + [item in list1 for item in list2])) -def set_direction_names(x=None, y=None, z=None): - """ - Sets the directional wildcards 'x', 'y', and 'z' to match the provided - dimension names only. - """ - for key, value in [('x', x), ('y', y), ('z', z)]: - if isinstance(value, string_types): - dim_names[key] = [key, value] - elif value is not None: - dim_names[key] = [key] + list(value) - - -def add_direction_names(x=None, y=None, z=None): - """ - Sets the directional wildcards 'x', 'y', and 'z' to match the provided - dimension names, in addition to any names they are already matching. - """ - for key, value in [('x', x), ('y', y), ('z', z)]: - if isinstance(value, string_types): - dim_names[key].append(value) - elif value is not None: - dim_names[key].extend(value) - - -def get_direction_name(dims, direction): - for name in dims: - if name in dim_names[direction] or name == direction: - return name - return None - - def combine_dims(dims1, dims2): """ Takes in two dims specifications and returns a single specification that @@ -513,33 +480,12 @@ def combine_dims(dims1, dims2): if dims1 == dims2: return dims1 dims_out = [] - for direction in dim_names.keys(): - dir1 = get_direction_name(dims1, direction) - dir2 = get_direction_name(dims2, direction) - if dir1 is None and dir2 is None: - pass - elif dir1 is None and dir2 is not None: - dims_out.append(dir2) - elif dir1 is not None and dir2 is None: - dims_out.append(dir1) - elif dir1 == direction: - dims_out.append(dir2) - elif dir2 == direction: - dims_out.append(dir1) - elif dir1 == dir2: - dims_out.append(dir1) - else: - raise InvalidPropertyDictError( - 'Two dims {} and {} exist for direction {}'.format( - dir1, dir2, direction)) - dims1 = set(dims1).difference(dims_out) - dims2 = set(dims2).difference(dims_out) + dims1 = set(dims1) + dims2 = set(dims2) dims1_wildcard = '*' in dims1 dims1.discard('*') - dims1 = dims1.difference(dim_names.keys()) dims2_wildcard = '*' in dims2 dims2.discard('*') - dims2 = dims2.difference(dim_names.keys()) unmatched_dims = set(dims1).union(dims2).difference(dims_out) shared_dims = set(dims2).intersection(dims2) if dims1_wildcard and dims2_wildcard: @@ -563,60 +509,6 @@ def combine_dims(dims1, dims2): return dims_out -def combine_array_dimensions(arrays, out_dims): - """ - Returns a tuple of dimension names corresponding to - dimension names from the DataArray objects given by *args when present. - The names returned correspond to the directions in out_dims. - - Args - ---- - arrays : iterable of DataArray - Objects from which to deduce dimension names. - out_dims : {'x', 'y', 'z'} - The desired output directions. Should contain only 'x', 'y', or 'z'. - For example, ('y', 'x') is valid. - - Raises - ------ - ValueError - If there are multiple names for a single direction, or if - an array has a dimension along a direction not present in out_dims. - - Returns - ------- - dimensions : list of str - The deduced dimension names, in the order given by out_dims. - """ - _ensure_no_invalid_directions(out_dims) - out_names = [None for _ in range(len(out_dims))] - all_names = set() - for value in arrays: - all_names.update(value.dims) - for direction, dir_names in dim_names.items(): - if direction in out_dims: - names = all_names.intersection(dir_names) - if len(names) > 1: - raise ValueError( - 'Multiple dimensions along {} direction'.format(direction)) - elif len(names) == 1: - out_names[out_dims.index(direction)] = names.pop() - else: - out_names[out_dims.index(direction)] = direction - elif len(all_names.intersection(dir_names)) > 0: - raise ValueError( - 'Arrays have dimensions along {} direction, which is ' - 'not included in output'.format(direction)) - return out_names - - -def _ensure_no_invalid_directions(out_dims): - invalid_dims = set(out_dims).difference(['x', 'y', 'z']) - if len(invalid_dims) != 0: - raise ValueError( - 'Invalid direction(s) in out_dims: {}'.format(invalid_dims)) - - def update_dict_by_adding_another(dict1, dict2): """ Takes two dictionaries. Add values in dict2 to the values in dict1, if diff --git a/tests/test_get_restore_numpy_array.py b/tests/test_get_restore_numpy_array.py index 2233bdf..fdb9c75 100644 --- a/tests/test_get_restore_numpy_array.py +++ b/tests/test_get_restore_numpy_array.py @@ -1,6 +1,6 @@ import pytest from sympl import ( - DataArray, set_direction_names, get_numpy_array, + DataArray, get_numpy_array, restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, InvalidStateError, InvalidPropertyDictError) @@ -206,26 +206,6 @@ def test_get_numpy_array_no_dimensions_listed_raises_value_error(): raise AssertionError('Expected ValueError but no error was raised') -def test_get_numpy_array_multiple_dims_on_same_direction(): - try: - set_direction_names(x=['lon']) - array = DataArray( - np.random.randn(2, 3), - dims=['x', 'lon'], - attrs={'units': ''}, - ) - try: - numpy_array = get_numpy_array(array, ['x', 'y']) - except ValueError: - pass - except Exception as err: - raise err - else: - raise AssertionError('Expected ValueError but no error was raised') - finally: - set_direction_names(x=[], y=[], z=[]) - - def test_get_numpy_array_not_enough_out_dims(): array = DataArray( np.random.randn(2, 3), @@ -481,12 +461,6 @@ def test_restore_dimensions_removes_dummy_axes(): class GetNumpyArraysWithPropertiesTests(unittest.TestCase): - def setUp(self): - pass - - def tearDown(self): - set_direction_names(x=(), y=(), z=()) - def test_returns_numpy_array(self): T_array = np.zeros([2, 3, 4], dtype=np.float64) + 280. property_dictionary = { @@ -604,59 +578,11 @@ def test_scalar_becomes_multidimensional_array(self): T_array) assert return_value['air_temperature'].base is T_array - def test_collects_wildcard_dimension(self): - set_direction_names(z=['mid_levels']) - T_array = np.zeros([2, 3, 4], dtype=np.float64) + 280. - property_dictionary = { - 'air_temperature': { - 'units': 'degK', - 'dims': ['x', 'y', 'z'], - }, - } - state = { - 'air_temperature': DataArray( - T_array, - dims=['x', 'y', 'mid_levels'], - attrs={'units': 'degK'}, - ), - } - return_value = get_numpy_arrays_with_properties(state, property_dictionary) - assert isinstance(return_value, dict) - assert len(return_value.keys()) == 1 - assert isinstance(return_value['air_temperature'], np.ndarray) - assert np.byte_bounds(return_value['air_temperature']) == np.byte_bounds( - T_array) - assert return_value['air_temperature'].base is T_array - assert return_value['air_temperature'].shape == (2, 3, 4) - - def test_raises_on_missing_explicit_dimension(self): - set_direction_names(z=['mid_levels']) - T_array = np.zeros([2, 3, 4], dtype=np.float64) + 280. - property_dictionary = { - 'air_temperature': { - 'units': 'degK', - 'dims': ['x', 'y', 'mid_levels'], - }, - } - state = { - 'air_temperature': DataArray( - T_array, - dims=['x', 'y', 'z'], - attrs={'units': 'degK'}, - ), - } - try: - return_value = get_numpy_arrays_with_properties(state, property_dictionary) - except InvalidStateError: - pass - else: - raise AssertionError('should have raised InvalidStateError') - def test_creates_length_1_dimensions(self): T_array = np.zeros([4], dtype=np.float64) + 280. property_dictionary = { 'air_temperature': { - 'dims': ['x', 'y', 'z'], + 'dims': ['*', 'z'], 'units': 'degK', }, } @@ -675,7 +601,7 @@ def test_creates_length_1_dimensions(self): return_value['air_temperature']) == np.byte_bounds( T_array) assert return_value['air_temperature'].base is T_array - assert return_value['air_temperature'].shape == (1, 1, 4) + assert return_value['air_temperature'].shape == (1, 4) def test_only_requested_properties_are_returned(self): property_dictionary = { @@ -904,100 +830,6 @@ def test_raises_if_dims_property_not_specified(self): else: raise AssertionError('should have raised ValueError') - def test_dims_like_accepts_valid_case(self): - set_direction_names(x=['x_cell_center', 'x_cell_interface'], - z=['mid_levels', 'interface_levels']) - property_dictionary = { - 'air_temperature': { - 'dims': ['x', 'y', 'mid_levels'], - 'units': 'degK', - }, - 'air_pressure': { - 'dims': ['x', 'y', 'interface_levels'], - 'units': 'Pa', - 'match_dims_like': 'air_temperature' - }, - } - state = { - 'air_temperature': DataArray( - np.zeros([2, 2, 4], dtype=np.float64), - dims=['x_cell_center', 'y', 'mid_levels'], - attrs={'units': 'degK'}, - ), - 'air_pressure': DataArray( - np.zeros([2, 2, 4], dtype=np.float64), - dims=['x_cell_center', 'y', 'interface_levels'], - attrs={'units': 'Pa'} - ), - } - return_value = get_numpy_arrays_with_properties(state, property_dictionary) - assert isinstance(return_value, dict) - assert len(return_value.keys()) == 2 - assert 'air_temperature' in return_value.keys() - assert 'air_pressure' in return_value.keys() - - def test_dims_like_rejects_mismatched_dimensions(self): - set_direction_names(x=['x_cell_center', 'x_cell_interface'], - z=['mid_levels', 'interface_levels']) - property_dictionary = { - 'air_temperature': { - 'dims': ['x', 'y', 'mid_levels'], - 'units': 'degK', - }, - 'air_pressure': { - 'dims': ['x', 'y', 'interface_levels'], - 'units': 'Pa', - 'match_dims_like': 'air_temperature' - }, - } - state = { - 'air_temperature': DataArray( - np.zeros([2, 2, 4], dtype=np.float64), - dims=['x_cell_center', 'y', 'mid_levels'], - attrs={'units': 'degK'}, - ), - 'air_pressure': DataArray( - np.zeros([2, 2, 4], dtype=np.float64), - dims=['x_cell_interface', 'y', 'interface_levels'], - attrs={'units': 'Pa'} - ), - } - try: - get_numpy_arrays_with_properties(state, property_dictionary) - except InvalidStateError: - pass - else: - raise AssertionError('should have raised InvalidStateError') - - def test_dims_like_raises_if_quantity_not_in_property_dict(self): - set_direction_names(x=['x_cell_center', 'x_cell_interface'], - z=['mid_levels', 'interface_levels']) - property_dictionary = { - 'air_pressure': { - 'dims': ['x', 'y', 'interface_levels'], - 'units': 'Pa', - 'match_dims_like': 'air_temperature' - }, - } - state = { - 'air_temperature': DataArray( - np.zeros([2, 2, 4], dtype=np.float64), - dims=['x_cell_center', 'y', 'mid_levels'], - attrs={'units': 'degK'}, - ), - 'air_pressure': DataArray( - np.zeros([2, 2, 4], dtype=np.float64), - dims=['x_cell_interface', 'y', 'interface_levels'], - attrs={'units': 'Pa'} - ), - } - try: - get_numpy_arrays_with_properties(state, property_dictionary) - except InvalidPropertyDictError: - pass - else: - raise AssertionError('should have raised InvalidPropertyDictError') - def test_collects_horizontal_dimensions(self): random = np.random.RandomState(0) T_array = random.randn(3, 2, 4) @@ -1070,12 +902,6 @@ def test_raises_when_quantity_has_extra_dim_and_unmatched_wildcard(self): class RestoreDataArraysWithPropertiesTests(unittest.TestCase): - def setUp(self): - pass - - def tearDown(self): - set_direction_names(x=(), y=(), z=()) - def test_restores_with_dims(self): raw_arrays = { 'output1': np.ones([10]), @@ -1250,50 +1076,6 @@ def test_restores_coords(self): assert return_value['air_temperature_tendency'].coords['z'].attrs['units'] == 'cm' assert return_value['air_temperature_tendency'].dims == input_state['air_temperature'].dims - def test_restores_matched_coords(self): - set_direction_names(x=['lon'], y=['lat'], z=['height']) - x = np.array([0., 10.]) - y = np.array([0., 10.]) - z = np.array([0., 5., 10., 15.]) - input_state = { - 'air_temperature': DataArray( - np.zeros([2, 2, 4]), - dims=['lon', 'lat', 'height'], - attrs={'units': 'degK'}, - coords=[ - ('lon', x, {'units': 'degrees_E'}), - ('lat', y, {'units': 'degrees_N'}), - ('height', z, {'units': 'km'})] - ) - } - input_properties = { - 'air_temperature': { - 'dims': ['x', 'y', 'z'], - 'units': 'degK', - } - } - raw_arrays = get_numpy_arrays_with_properties(input_state, input_properties) - raw_arrays = {key + '_tendency': value for key, value in raw_arrays.items()} - output_properties = { - 'air_temperature_tendency': { - 'dims_like': 'air_temperature', - 'units': 'degK/s', - } - } - return_value = restore_data_arrays_with_properties( - raw_arrays, output_properties, input_state, input_properties - ) - assert np.all(return_value['air_temperature_tendency'].coords['lon'] == - input_state['air_temperature'].coords['lon']) - assert return_value['air_temperature_tendency'].coords['lon'].attrs['units'] == 'degrees_E' - assert np.all(return_value['air_temperature_tendency'].coords['lat'] == - input_state['air_temperature'].coords['lat']) - assert return_value['air_temperature_tendency'].coords['lat'].attrs['units'] == 'degrees_N' - assert np.all(return_value['air_temperature_tendency'].coords['height'] == - input_state['air_temperature'].coords['height']) - assert return_value['air_temperature_tendency'].coords['height'].attrs['units'] == 'km' - assert return_value['air_temperature_tendency'].dims == input_state['air_temperature'].dims - def test_restores_scalar_array(self): T_array = np.array(0.) input_properties = { diff --git a/tests/test_match_dims_like.py b/tests/test_match_dims_like.py index 9be29b3..3fdd6e1 100644 --- a/tests/test_match_dims_like.py +++ b/tests/test_match_dims_like.py @@ -1,11 +1,7 @@ import pytest from sympl import ( - DataArray, set_direction_names, get_numpy_array, - restore_dimensions, get_numpy_arrays_with_properties, - restore_data_arrays_with_properties, InvalidStateError, - InvalidPropertyDictError) + DataArray, get_numpy_arrays_with_properties, InvalidStateError) import numpy as np -import unittest def test_match_dims_like_hardcoded_dimensions_matching_lengths(): @@ -185,73 +181,5 @@ def test_match_dims_like_wildcard_dimensions_use_same_ordering(): assert np.all(raw_arrays['air_temperature'] == raw_arrays['air_pressure']) -class MultipleWildcardDimensionsTests(unittest.TestCase): - - def setUp(self): - pass - - def tearDown(self): - set_direction_names(x=(), y=(), z=()) - - def test_match_dims_like_x_y_z_matching_lengths(self): - set_direction_names(x=['lat'], y=['lon'], z=['mid_levels', 'interface_levels']) - input_state = { - 'air_temperature': DataArray( - np.zeros([2, 3, 4]), - dims=['lat', 'lon', 'mid_levels'], - attrs={'units': 'degK'}, - ), - 'air_pressure': DataArray( - np.zeros([2, 3, 4]), - dims=['lat', 'lon', 'mid_levels'], - attrs={'units': 'Pa'}, - ), - } - input_properties = { - 'air_temperature': { - 'dims': ['x', 'y', 'z'], - 'units': 'degK', - 'match_dims_like': 'air_pressure', - }, - 'air_pressure': { - 'dims': ['x', 'y', 'z'], - 'units': 'Pa', - }, - } - raw_arrays = get_numpy_arrays_with_properties( - input_state, input_properties) - assert np.byte_bounds(raw_arrays['air_temperature']) == np.byte_bounds(input_state['air_temperature'].values) - assert np.byte_bounds(raw_arrays['air_pressure']) == np.byte_bounds(input_state['air_pressure'].values) - - def test_match_dims_like_star_z_matching_lengths(self): - set_direction_names(x=['lat'], y=['lon'], z=['mid_levels', 'interface_levels']) - input_state = { - 'air_temperature': DataArray( - np.zeros([2, 3, 4]), - dims=['lat', 'lon', 'interface_levels'], - attrs={'units': 'degK'}, - ), - 'air_pressure': DataArray( - np.zeros([2, 3, 4]), - dims=['lat', 'lon', 'interface_levels'], - attrs={'units': 'Pa'}, - ), - } - input_properties = { - 'air_temperature': { - 'dims': ['*', 'z'], - 'units': 'degK', - 'match_dims_like': 'air_pressure', - }, - 'air_pressure': { - 'dims': ['*', 'z'], - 'units': 'Pa', - }, - } - raw_arrays = get_numpy_arrays_with_properties( - input_state, input_properties) - assert np.byte_bounds(raw_arrays['air_temperature']) == np.byte_bounds(input_state['air_temperature'].values) - assert np.byte_bounds(raw_arrays['air_pressure']) == np.byte_bounds(input_state['air_pressure'].values) - if __name__ == '__main__': pytest.main([__file__]) diff --git a/tests/test_util.py b/tests/test_util.py index f334e10..26d7131 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -3,7 +3,7 @@ import pytest from sympl import ( Prognostic, ensure_no_shared_keys, SharedKeyError, DataArray, - combine_array_dimensions, set_direction_names, Implicit, Diagnostic, + Implicit, Diagnostic, InvalidPropertyDictError) from sympl._core.util import update_dict_by_adding_another, combine_dims, get_component_aliases @@ -173,72 +173,8 @@ def test_ensure_no_shared_keys_with_shared_keys(): 'No exception raised but expected SharedKeyError.') -class CombineArrayDimensionsTests(unittest.TestCase): - - def setUp(self): - self.array_1d = DataArray(np.zeros((2,)), dims=['lon']) - self.array_2d = DataArray(np.zeros((2, 2)), dims=['lat', 'lon']) - self.array_3d = DataArray(np.zeros((2, 2, 2)), - dims=['lon', 'lat', 'interface_levels']) - set_direction_names( - x=['lon'], y=['lat'], z=['mid_levels', 'interface_levels']) - - def tearDown(self): - set_direction_names(x=[], y=[], z=[]) - - def test_combine_dimensions_2d_and_3d(self): - dims = combine_array_dimensions( - [self.array_2d, self.array_3d], out_dims=('x', 'y', 'z')) - assert same_list(dims, ['lon', 'lat', 'interface_levels']) - - def test_combine_dimensions_2d_and_3d_z_y_x(self): - dims = combine_array_dimensions( - [self.array_2d, self.array_3d], out_dims=('z', 'y', 'x')) - assert same_list(dims, ['interface_levels', 'lat', 'lon']) - - def combine_dimensions_1d_shared(self): - dims = combine_array_dimensions( - [self.array_1d, self.array_1d], out_dims=['x']) - assert same_list(dims, ['lon']) - - def combine_dimensions_1d_not_shared(self): - array_1d_x = DataArray(np.zeros((2,)), dims=['lon']) - array_1d_y = DataArray(np.zeros((2,)), dims=['lat']) - dims = combine_array_dimensions([array_1d_x, array_1d_y], out_dims=['x', 'y']) - assert same_list(dims, ['lon', 'lat']) - - def combine_dimensions_1d_wrong_direction(self): - try: - combine_array_dimensions( - [self.array_1d, self.array_1d], out_dims=['z']) - except ValueError: - pass - except Exception as err: - raise err - else: - raise AssertionError('No exception raised but expected ValueError.') - - def combine_dimensions_1d_and_2d_extra_direction(self): - try: - combine_array_dimensions( - [self.array_1d, self.array_2d], out_dims=['y']) - except ValueError: - pass - except Exception as err: - raise err - else: - raise AssertionError('No exception raised but expected ValueError.') - - class CombineDimsTests(unittest.TestCase): - def setUp(self): - set_direction_names( - x=['lon'], y=['lat'], z=['mid_levels', 'interface_levels']) - - def tearDown(self): - set_direction_names(x=[], y=[], z=[]) - def test_same_dims(self): assert combine_dims(['dim1'], ['dim1']) == ['dim1'] @@ -266,24 +202,6 @@ def test_swapped_dims_with_wildcard(self): result = combine_dims(['*', 'dim1', 'dim2'], ['*', 'dim2', 'dim1']) assert set(result) == {'*', 'dim1', 'dim2'} - def test_one_directional(self): - set_direction_names(x=['lon']) - assert combine_dims(['lon'], ['lon']) == ['lon'] - - def test_overlapping_directional(self): - set_direction_names(x=['lon_mid', 'lon_edge']) - with self.assertRaises(InvalidPropertyDictError): - combine_dims(['lon_mid'], ['lon_edge']) - - def test_combine_directional_wildcard(self): - set_direction_names(x=['lon']) - assert combine_dims(['x'], ['lon']) == ['lon'] - - def test_combine_multiple_directional_wildcard(self): - set_direction_names(x=['lon']) - result = combine_dims(['*', 'x', 'y'], ['*', 'lon']) - assert set(result) == {'*', 'lon', 'y'} - if __name__ == '__main__': pytest.main([__file__]) From 85c8a60117b8e6811f801f65cf5c283480ef64bb Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 22 Mar 2018 23:38:29 -0700 Subject: [PATCH 18/98] flake8 fixes --- sympl/__init__.py | 2 +- sympl/_core/composite.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sympl/__init__.py b/sympl/__init__.py index bad30f3..51a532c 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -35,7 +35,7 @@ InvalidPropertyDictError, DataArray, get_constant, set_constant, set_condensible_name, reset_constants, - get_constants_string,TimeDifferencingWrapper, + get_constants_string, TimeDifferencingWrapper, ensure_no_shared_keys, get_numpy_array, jit, restore_dimensions, get_numpy_arrays_with_properties, diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index ba50e8a..85c3386 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -183,6 +183,7 @@ def __call__(self, state): def array_call(self, state): raise NotImplementedError() + class MonitorComposite(ComponentComposite): component_class = Monitor From 06e8176da938f753eb56f396194733bd5701e643 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 23 Mar 2018 11:40:06 -0700 Subject: [PATCH 19/98] Added tests for composite properties --- tests/test_composite.py | 866 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 817 insertions(+), 49 deletions(-) diff --git a/tests/test_composite.py b/tests/test_composite.py index 775b3aa..2e411d4 100644 --- a/tests/test_composite.py +++ b/tests/test_composite.py @@ -2,8 +2,9 @@ import mock from sympl import ( Prognostic, Diagnostic, Monitor, PrognosticComposite, DiagnosticComposite, - MonitorComposite, SharedKeyError, DataArray + MonitorComposite, SharedKeyError, DataArray, InvalidPropertyDictError ) +from sympl._core.units import units_are_compatible def same_list(list1, list2): @@ -13,37 +14,82 @@ def same_list(list1, list2): class MockPrognostic(Prognostic): + input_properties = None + diagnostic_properties = None + tendency_properties = None + + def __init__( + self, input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, **kwargs): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self.tendency_properties = tendency_properties + self._diagnostic_output = diagnostic_output + self._tendency_output = tendency_output + self.times_called = 0 + self.state_given = None + super(MockPrognostic, self).__init__(**kwargs) + + def array_call(self, state): + self.times_called += 1 + self.state_given = state + return self._tendency_output, self._diagnostic_output + + +class MockDiagnostic(Diagnostic): + + input_properties = None + diagnostic_properties = None + + def __init__( + self, input_properties, diagnostic_properties, diagnostic_output, + **kwargs): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self._diagnostic_output = diagnostic_output + self.times_called = 0 + self.state_given = None + super(MockDiagnostic, self).__init__(**kwargs) + + def array_call(self, state): + self.times_called += 1 + self.state_given = state + return self._diagnostic_output + + +class MockEmptyPrognostic(Prognostic): + input_properties = {} diagnostic_properties = {} tendency_properties = {} def __init__(self, **kwargs): - super(MockPrognostic, self).__init__(**kwargs) + super(MockEmptyPrognostic, self).__init__(**kwargs) def array_call(self, state): return {}, {} -class MockPrognostic2(Prognostic): +class MockEmptyPrognostic2(Prognostic): input_properties = {} diagnostic_properties = {} tendency_properties = {} def __init__(self, **kwargs): - super(MockPrognostic2, self).__init__(**kwargs) + super(MockEmptyPrognostic2, self).__init__(**kwargs) def array_call(self, state): return {}, {} -class MockDiagnostic(Diagnostic): +class MockEmptyDiagnostic(Diagnostic): input_properties = {} diagnostic_properties = {} def __init__(self, **kwargs): - super(MockDiagnostic, self).__init__(**kwargs) + super(MockEmptyDiagnostic, self).__init__(**kwargs) def array_call(self, state): return {} @@ -65,10 +111,10 @@ def test_empty_prognostic_composite(): assert isinstance(diagnostics, dict) -@mock.patch.object(MockPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognostic, '__call__') def test_prognostic_composite_calls_one_prognostic(mock_call): mock_call.return_value = ({'air_temperature': 0.5}, {'foo': 50.}) - prognostic_composite = PrognosticComposite(MockPrognostic()) + prognostic_composite = PrognosticComposite(MockEmptyPrognostic()) state = {'air_temperature': 273.15} tendencies, diagnostics = prognostic_composite(state) assert mock_call.called @@ -76,11 +122,11 @@ def test_prognostic_composite_calls_one_prognostic(mock_call): assert diagnostics == {'foo': 50.} -@mock.patch.object(MockPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognostic, '__call__') def test_prognostic_composite_calls_two_prognostics(mock_call): mock_call.return_value = ({'air_temperature': 0.5}, {}) prognostic_composite = PrognosticComposite( - MockPrognostic(), MockPrognostic()) + MockEmptyPrognostic(), MockEmptyPrognostic()) state = {'air_temperature': 273.15} tendencies, diagnostics = prognostic_composite(state) assert mock_call.called @@ -97,10 +143,10 @@ def test_empty_diagnostic_composite(): assert isinstance(diagnostics, dict) -@mock.patch.object(MockDiagnostic, '__call__') +@mock.patch.object(MockEmptyDiagnostic, '__call__') def test_diagnostic_composite_calls_one_diagnostic(mock_call): mock_call.return_value = {'foo': 50.} - diagnostic_composite = DiagnosticComposite(MockDiagnostic()) + diagnostic_composite = DiagnosticComposite(MockEmptyDiagnostic()) state = {'air_temperature': 273.15} diagnostics = diagnostic_composite(state) assert mock_call.called @@ -135,7 +181,7 @@ def test_monitor_collection_calls_two_monitors(mock_store): def test_prognostic_composite_cannot_use_diagnostic(): try: - PrognosticComposite(MockDiagnostic()) + PrognosticComposite(MockEmptyDiagnostic()) except TypeError: pass except Exception as err: @@ -146,7 +192,7 @@ def test_prognostic_composite_cannot_use_diagnostic(): def test_diagnostic_composite_cannot_use_prognostic(): try: - DiagnosticComposite(MockPrognostic()) + DiagnosticComposite(MockEmptyPrognostic()) except TypeError: pass except Exception as err: @@ -155,11 +201,11 @@ def test_diagnostic_composite_cannot_use_prognostic(): raise AssertionError('TypeError should have been raised') -@mock.patch.object(MockDiagnostic, '__call__') +@mock.patch.object(MockEmptyDiagnostic, '__call__') def test_diagnostic_composite_call(mock_call): mock_call.return_value = {'foo': 5.} state = {'bar': 10.} - diagnostics = DiagnosticComposite(MockDiagnostic()) + diagnostics = DiagnosticComposite(MockEmptyDiagnostic()) new_state = diagnostics(state) assert list(state.keys()) == ['bar'] assert state['bar'] == 10. @@ -167,54 +213,776 @@ def test_diagnostic_composite_call(mock_call): assert new_state['foo'] == 5. -def test_prognostic_composite_ensures_valid_state(): - prognostic1 = MockPrognostic() - prognostic1.input_properties = {'input1': {}} - prognostic1.diagnostic_properties = {'diagnostic1': {}} - prognostic2 = MockPrognostic() - prognostic2.input_properties = {'input1': {}, 'input2': {}} - prognostic2.diagnostic_properties = {'diagnostic1': {}} - try: - PrognosticComposite(prognostic1, prognostic2) - except SharedKeyError: - pass - except Exception as err: - raise err - else: - raise AssertionError( - 'Should not be able to have overlapping diagnostics in composite') - - -@mock.patch.object(MockPrognostic, '__call__') -@mock.patch.object(MockPrognostic2, '__call__') +@mock.patch.object(MockEmptyPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognostic2, '__call__') def test_prognostic_component_handles_units_when_combining(mock_call, mock2_call): mock_call.return_value = ({ 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})}, {}) mock2_call.return_value = ({ 'eastward_wind': DataArray(50., attrs={'units': 'cm/s'})}, {}) - prognostic1 = MockPrognostic() - prognostic2 = MockPrognostic2() + prognostic1 = MockEmptyPrognostic() + prognostic2 = MockEmptyPrognostic2() composite = PrognosticComposite(prognostic1, prognostic2) tendencies, diagnostics = composite({}) assert tendencies['eastward_wind'].to_units('m/s').values.item() == 1.5 -def test_diagnostic_composite_ensures_valid_state(): - diagnostic1 = MockDiagnostic() - diagnostic1.input_properties = {'input1': {}} - diagnostic1.diagnostic_properties = {'diagnostic1': {}} - diagnostic2 = MockDiagnostic() - diagnostic2.input_properties = {'input1': {}, 'input2': {}} - diagnostic2.diagnostic_properties = {'diagnostic1': {}} +def test_diagnostic_composite_single_component_input(): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + }, + 'input2': { + 'dims': ['dim2'], + 'units': 'm/s' + }, + } + diagnostic_properties = {} + diagnostic_output = {} + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, diagnostic_output) + composite = DiagnosticComposite(diagnostic) + assert composite.input_properties == input_properties + assert composite.diagnostic_properties == diagnostic_properties + + +def test_diagnostic_composite_single_component_diagnostic(): + input_properties = {} + diagnostic_properties = { + 'diag1': { + 'dims': ['lon'], + 'units': 'km', + }, + 'diag2': { + 'dims': ['lon'], + 'units': 'degK', + }, + } + diagnostic_output = {} + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, diagnostic_output) + composite = DiagnosticComposite(diagnostic) + assert composite.input_properties == input_properties + assert composite.diagnostic_properties == diagnostic_properties + + +def test_diagnostic_composite_single_empty_component(): + input_properties = {} + diagnostic_properties = {} + diagnostic_output = {} + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, diagnostic_output) + composite = DiagnosticComposite(diagnostic) + assert composite.input_properties == input_properties + assert composite.diagnostic_properties == diagnostic_properties + + +def test_diagnostic_composite_single_full_component(): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + }, + 'input2': { + 'dims': ['dim2'], + 'units': 'm/s' + }, + } + diagnostic_properties = { + 'diag1': { + 'dims': ['lon'], + 'units': 'km', + }, + 'diag2': { + 'dims': ['lon'], + 'units': 'degK', + }, + } + diagnostic_output = {} + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, diagnostic_output) + composite = DiagnosticComposite(diagnostic) + assert composite.input_properties == input_properties + assert composite.diagnostic_properties == diagnostic_properties + + +def test_diagnostic_composite_two_components_no_overlap(): + diagnostic1 = MockDiagnostic( + input_properties={ + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + }, + }, + diagnostic_properties={ + 'diag1': { + 'dims': ['lon'], + 'units': 'km', + }, + }, + diagnostic_output={} + ) + diagnostic2 = MockDiagnostic( + input_properties={ + 'input2': { + 'dims': ['dim2'], + 'units': 'm/s' + }, + }, + diagnostic_properties={ + 'diag2': { + 'dims': ['lon'], + 'units': 'degK', + }, + }, + diagnostic_output={} + ) + composite = DiagnosticComposite(diagnostic1, diagnostic2) + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + }, + 'input2': { + 'dims': ['dim2'], + 'units': 'm/s' + }, + } + diagnostic_properties = { + 'diag1': { + 'dims': ['lon'], + 'units': 'km', + }, + 'diag2': { + 'dims': ['lon'], + 'units': 'degK', + }, + } + assert composite.input_properties == input_properties + assert composite.diagnostic_properties == diagnostic_properties + + +def test_diagnostic_composite_two_components_overlap_input(): + diagnostic1 = MockDiagnostic( + input_properties={ + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + }, + 'input2': { + 'dims': ['dim2'], + 'units': 'm/s' + }, + }, + diagnostic_properties={ + 'diag1': { + 'dims': ['lon'], + 'units': 'km', + }, + }, + diagnostic_output={} + ) + diagnostic2 = MockDiagnostic( + input_properties={ + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + }, + 'input2': { + 'dims': ['dim2'], + 'units': 'm/s' + }, + }, + diagnostic_properties={ + 'diag2': { + 'dims': ['lon'], + 'units': 'degK', + }, + }, + diagnostic_output={} + ) + composite = DiagnosticComposite(diagnostic1, diagnostic2) + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + }, + 'input2': { + 'dims': ['dim2'], + 'units': 'm/s' + }, + } + diagnostic_properties = { + 'diag1': { + 'dims': ['lon'], + 'units': 'km', + }, + 'diag2': { + 'dims': ['lon'], + 'units': 'degK', + }, + } + assert composite.input_properties == input_properties + assert composite.diagnostic_properties == diagnostic_properties + + +def test_diagnostic_composite_two_components_overlap_diagnostic(): + diagnostic1 = MockDiagnostic( + input_properties={}, + diagnostic_properties={ + 'diag1': { + 'dims': ['lon'], + 'units': 'km', + }, + }, + diagnostic_output={} + ) + diagnostic2 = MockDiagnostic( + input_properties={}, + diagnostic_properties={ + 'diag1': { + 'dims': ['lon'], + 'units': 'km', + }, + }, + diagnostic_output={} + ) try: DiagnosticComposite(diagnostic1, diagnostic2) except SharedKeyError: pass - except Exception as err: - raise err else: - raise AssertionError( - 'Should not be able to have overlapping diagnostics in composite') + raise AssertionError('Should have raised SharedKeyError') + + +def test_diagnostic_composite_two_components_incompatible_input_dims(): + diagnostic1 = MockDiagnostic( + input_properties={ + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + } + }, + diagnostic_properties={}, + diagnostic_output={} + ) + diagnostic2 = MockDiagnostic( + input_properties={ + 'input1': { + 'dims': ['dim2'], + 'units': 'm', + } + }, + diagnostic_properties={}, + diagnostic_output={} + ) + try: + composite = DiagnosticComposite(diagnostic1, diagnostic2) + except InvalidPropertyDictError: + pass + else: + raise AssertionError('Should have raised InvalidPropertyDictError') + + +def test_diagnostic_composite_two_components_incompatible_input_units(): + diagnostic1 = MockDiagnostic( + input_properties={ + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + } + }, + diagnostic_properties={}, + diagnostic_output={} + ) + diagnostic2 = MockDiagnostic( + input_properties={ + 'input1': { + 'dims': ['dim1'], + 'units': 's', + } + }, + diagnostic_properties={}, + diagnostic_output={} + ) + try: + DiagnosticComposite(diagnostic1, diagnostic2) + except InvalidPropertyDictError: + pass + else: + raise AssertionError('Should have raised InvalidPropertyDictError') + + +def test_prognostic_composite_single_input(): + prognostic = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1'], + 'units': 'm', + } + }, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic) + assert composite.input_properties == prognostic.input_properties + assert composite.diagnostic_properties == prognostic.diagnostic_properties + assert composite.tendency_properties == prognostic.tendency_properties + + +def test_prognostic_composite_single_diagnostic(): + prognostic = MockPrognostic( + input_properties={}, + diagnostic_properties={ + 'diag1': { + 'dims': ['dims2'], + 'units': '', + } + }, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic) + assert composite.input_properties == prognostic.input_properties + assert composite.diagnostic_properties == prognostic.diagnostic_properties + assert composite.tendency_properties == prognostic.tendency_properties + + +def test_prognostic_composite_single_tendency(): + prognostic = MockPrognostic( + input_properties={}, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic) + assert composite.input_properties == prognostic.input_properties + assert composite.diagnostic_properties == prognostic.diagnostic_properties + assert composite.tendency_properties == prognostic.tendency_properties + + +def test_prognostic_composite_two_components_input(): + prognostic1 = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK', + }, + 'input2': { + 'dims': ['dims1'], + 'units': 'm', + }, + }, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK', + }, + 'input3': { + 'dims': ['dims2'], + 'units': '', + }, + }, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic1, prognostic2) + input_properties = { + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK', + }, + 'input2': { + 'dims': ['dims1'], + 'units': 'm', + }, + 'input3': { + 'dims': ['dims2'], + 'units': '', + }, + } + diagnostic_properties = {} + tendency_properties = {} + assert composite.input_properties == input_properties + assert composite.diagnostic_properties == diagnostic_properties + assert composite.tendency_properties == tendency_properties + + +def test_prognostic_composite_two_components_swapped_input_dims(): + prognostic1 = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK', + }, + }, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims2', 'dims1'], + 'units': 'degK', + }, + }, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic1, prognostic2) + diagnostic_properties = {} + tendency_properties = {} + assert (composite.input_properties == prognostic1.input_properties or + composite.input_properties == prognostic2.input_properties) + assert composite.diagnostic_properties == diagnostic_properties + assert composite.tendency_properties == tendency_properties + + +def test_prognostic_composite_two_components_incompatible_input_dims(): + prognostic1 = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK', + }, + }, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims3'], + 'units': 'degK', + }, + }, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + try: + PrognosticComposite(prognostic1, prognostic2) + except InvalidPropertyDictError: + pass + else: + raise AssertionError('Should have raised InvalidPropertyDictError') + + +def test_prognostic_composite_two_components_incompatible_input_units(): + prognostic1 = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK', + }, + }, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'm', + }, + }, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + try: + PrognosticComposite(prognostic1, prognostic2) + except InvalidPropertyDictError: + pass + else: + raise AssertionError('Should have raised InvalidPropertyDictError') + + +def test_prognostic_composite_two_components_compatible_input_units(): + prognostic1 = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'km', + }, + }, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'cm', + }, + }, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic1, prognostic2) + assert 'input1' in composite.input_properties.keys() + assert composite.input_properties['input1']['dims'] == ['dims1', 'dims2'] + assert units_are_compatible(composite.input_properties['input1']['units'], 'm') + + +def test_prognostic_composite_two_components_tendency(): + prognostic1 = MockPrognostic( + input_properties={}, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dim1'], + 'units': 'm/s', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={}, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dim1'], + 'units': 'm/s' + }, + 'tend2': { + 'dims': ['dim1'], + 'units': 'degK/s' + } + }, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic1, prognostic2) + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'tend1': { + 'dims': ['dim1'], + 'units': 'm/s' + }, + 'tend2': { + 'dims': ['dim1'], + 'units': 'degK/s' + } + } + assert composite.input_properties == input_properties + assert composite.diagnostic_properties == diagnostic_properties + assert composite.tendency_properties == tendency_properties + + +def test_prognostic_composite_two_components_tendency_incompatible_dims(): + prognostic1 = MockPrognostic( + input_properties={}, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dim1'], + 'units': 'm/s', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={}, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dim2'], + 'units': 'm/s' + }, + 'tend2': { + 'dims': ['dim1'], + 'units': 'degK/s' + } + }, + diagnostic_output={}, + tendency_output={}, + ) + try: + PrognosticComposite(prognostic1, prognostic2) + except InvalidPropertyDictError: + pass + else: + raise AssertionError('Should have raised InvalidPropertyDictError') + + +def test_prognostic_composite_two_components_tendency_incompatible_units(): + prognostic1 = MockPrognostic( + input_properties={}, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dim1'], + 'units': 'm/s', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={}, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dim1'], + 'units': 'degK/s' + }, + 'tend2': { + 'dims': ['dim1'], + 'units': 'degK/s' + } + }, + diagnostic_output={}, + tendency_output={}, + ) + try: + PrognosticComposite(prognostic1, prognostic2) + except InvalidPropertyDictError: + pass + else: + raise AssertionError('Should have raised InvalidPropertyDictError') + + +def test_prognostic_composite_two_components_tendency_compatible_units(): + prognostic1 = MockPrognostic( + input_properties={}, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dim1'], + 'units': 'km/s', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={}, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dim1'], + 'units': 'm/day' + }, + }, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic1, prognostic2) + assert 'tend1' in composite.tendency_properties.keys() + assert composite.tendency_properties['tend1']['dims'] == ['dim1'] + assert units_are_compatible(composite.tendency_properties['tend1']['units'], 'm/s') + + +def test_prognostic_composite_two_components_diagnostic(): + prognostic1 = MockPrognostic( + input_properties={}, + diagnostic_properties={ + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + } + }, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={}, + diagnostic_properties={ + 'diag2': { + 'dims': ['dim2'], + 'units': 'm', + } + }, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic1, prognostic2) + input_properties = {} + diagnostic_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + }, + 'diag2': { + 'dims': ['dim2'], + 'units': 'm', + }, + } + tendency_properties = {} + assert composite.input_properties == input_properties + assert composite.diagnostic_properties == diagnostic_properties + assert composite.tendency_properties == tendency_properties + + +def test_prognostic_composite_two_components_overlapping_diagnostic(): + prognostic1 = MockPrognostic( + input_properties={}, + diagnostic_properties={ + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + } + }, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={}, + diagnostic_properties={ + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + } + }, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + ) + try: + PrognosticComposite(prognostic1, prognostic2) + except SharedKeyError: + pass + else: + raise AssertionError('Should have raised SharedKeyError') if __name__ == '__main__': From 9d39ec3d2dc12789f93402ebf2c92021f87309c1 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 23 Mar 2018 13:40:25 -0700 Subject: [PATCH 20/98] Updated basic components to use new component API --- HISTORY.rst | 4 + sympl/_components/basic.py | 212 ++++++++++++++++++++------------- sympl/_core/base_components.py | 38 +++--- tests/test_components.py | 147 ++++++++++++++++------- 4 files changed, 254 insertions(+), 147 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 51dada0..22a500b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ Latest * TimeSteppers now allow you to put tendencies in their diagnostic output. This is done using first-order time differencing. * Composites now have properties dictionaries. +* Updated basic components to use new component API. Breaking changes ~~~~~~~~~~~~~~~~ @@ -48,6 +49,9 @@ Breaking changes only wildcard dimension. 'x', 'y', and 'z' refer to their own names only. * Removed the combine_dimensions function, which wasn't used anywhere and no longer has much purpose without directional wildcards +* RelaxationPrognostic no longer allows caching of equilibrium values or + timescale. They must be provided through the input state. This is to ensure + proper conversion of dimensions and units. v0.3.2 ------ diff --git a/sympl/_components/basic.py b/sympl/_components/basic.py index 55a426d..de697f2 100644 --- a/sympl/_components/basic.py +++ b/sympl/_components/basic.py @@ -1,6 +1,7 @@ from .._core.array import DataArray from .._core.base_components import ImplicitPrognostic, Prognostic, Diagnostic from .._core.units import unit_registry as ureg +from .._core.util import combine_dims class ConstantPrognostic(Prognostic): @@ -12,6 +13,30 @@ class ConstantPrognostic(Prognostic): it would also modify the values inside this object. """ + @property + def input_properties(self): + return {} + + @property + def tendency_properties(self): + return_dict = {} + for name, data_array in self.__tendencies.items(): + return_dict[name] = { + 'dims': data_array.dims, + 'units': data_array.attrs['units'], + } + return return_dict + + @property + def diagnostic_properties(self): + return_dict = {} + for name, data_array in self.__diagnostics.items(): + return_dict[name] = { + 'dims': data_array.dims, + 'units': data_array.attrs['units'], + } + return return_dict + def __init__(self, tendencies, diagnostics=None): """ Args @@ -25,34 +50,21 @@ def __init__(self, tendencies, diagnostics=None): state quantities and values are the value of those quantities to be returned by this Prognostic. """ - self._tendencies = tendencies.copy() + self.__tendencies = tendencies.copy() if diagnostics is not None: - self._diagnostics = diagnostics.copy() + self.__diagnostics = diagnostics.copy() else: - self._diagnostics = {} - - def __call__(self, state): - """ - Gets tendencies and diagnostics from the passed model state. The - returned dictionaries will contain the same values as were passed at - initialization. - - Args - ---- - state : dict - A model state dictionary. + self.__diagnostics = {} + super(ConstantPrognostic, self).__init__() - Returns - ------- - tendencies : dict - A dictionary whose keys are strings indicating - state quantities and values are the time derivative of those - quantities in units/second. - diagnostics : dict - A dictionary whose keys are strings indicating - state quantities and values are the value of those quantities. - """ - return self._tendencies.copy(), self._diagnostics.copy() + def array_call(self, state): + tendencies = {} + for name, data_array in self.__tendencies.items(): + tendencies[name] = data_array.values + diagnostics = {} + for name, data_array in self.__diagnostics.items(): + diagnostics[name] = data_array.values + return tendencies, diagnostics class ConstantDiagnostic(Diagnostic): @@ -66,6 +78,20 @@ class ConstantDiagnostic(Diagnostic): it would also modify the values inside this object. """ + @property + def input_properties(self): + return {} + + @property + def diagnostic_properties(self): + return_dict = {} + for name, data_array in self.__diagnostics.items(): + return_dict[name] = { + 'dims': data_array.dims, + 'units': data_array.attrs['units'], + } + return return_dict + def __init__(self, diagnostics): """ Args @@ -76,27 +102,14 @@ def __init__(self, diagnostics): The values in the dictionary will be returned when this Diagnostic is called. """ - self._diagnostics = diagnostics.copy() - - def __call__(self, state): - """ - Returns diagnostic values. - - Args - ---- - state : dict - A model state dictionary. Is not used, and is only - taken in to keep an API consistent with a Diagnostic. + self.__diagnostics = diagnostics.copy() + super(ConstantDiagnostic, self).__init__() - Returns - ------- - diagnostics : dict - A dictionary whose keys are strings indicating - state quantities and values are the value of those quantities. - The values in the returned dictionary are the same as were - passed into this object at initialization. - """ - return self._diagnostics.copy() + def array_call(self, state): + return_state = {} + for name, data_array in self.__diagnostics.items(): + return_state[name] = data_array.values + return return_state class RelaxationPrognostic(Prognostic): @@ -109,35 +122,77 @@ class RelaxationPrognostic(Prognostic): equilibrium value, and :math:`\tau` is the timescale of the relaxation. """ - def __init__(self, quantity_name, equilibrium_value=None, - relaxation_timescale=None): + @property + def input_properties(self): + return_dict = { + self._quantity_name: { + 'dims': ['*'], + 'units': self._units, + }, + 'equilibrium_{}'.format(self._quantity_name): { + 'dims': ['*'], + 'units': self._units, + }, + '{}_relaxation_timescale'.format(self._quantity_name): { + 'dims': ['*'], + 'units': 's', + } + } + return return_dict + + @property + def tendency_properties(self): + return { + self._quantity_name: { + 'dims': ['*'], + 'units': str(ureg(self._units) / ureg('s')), + } + } + + @property + def diagnostic_properties(self): + return {} + + def __init__(self, quantity_name, units, **kwargs): """ Args ---- quantity_name : str - The name of the quantity to which Newtonian - relaxation should be applied - equilibrium_value : DataArray, optional - The equilibrium value to which the quantity is relaxed. If - not given, it should be provided in the state when - the object is called. - relaxation_timescale : DataArray, optional - The timescale tau with which the Newtonian relaxation occurs. - If not given, it should be provided in the state when - the object is called. + The name of the quantity to which Newtonian relaxation should be + applied. + units : str + The units of the relaxed quantity as to be used internally when + computing tendency. Can be any units convertible from the actual + input you plan to use. + input_scale_factors : dict, optional + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + tendency_scale_factors : dict, optional + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which tendency values are scaled before being + returned by this object. + diagnostic_scale_factors : dict, optional + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + update_interval : timedelta, optional + If given, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. """ self._quantity_name = quantity_name - self._equilibrium_value = equilibrium_value - self._tau = relaxation_timescale + self._units = units + super(RelaxationPrognostic, self).__init__(**kwargs) - def __call__(self, state): + def array_call(self, state): """ Gets tendencies and diagnostics from the passed model state. Args ---- state : dict - A model state dictionary. Below, (quantity_name) + A model state dictionary as numpy arrays. Below, (quantity_name) refers to the quantity_name passed at initialization. The state must contain: @@ -152,33 +207,18 @@ def __call__(self, state): tendencies : dict A dictionary whose keys are strings indicating state quantities and values are the time derivative of those - quantities in units/second at the time of the input state. + quantities in units/second at the time of the input state, as + numpy arrays. diagnostics : dict A dictionary whose keys are strings indicating state quantities and values are the value of those quantities - at the time of the input state. + at the time of the input state, as numpy arrays. """ value = state[self._quantity_name] - if self._equilibrium_value is None: - equilibrium = state['equilibrium_' + self._quantity_name].to_units( - value.attrs['units']) - else: - equilibrium = self._equilibrium_value.to_units( - value.attrs['units']) - if self._tau is None: - tau = state[ - self._quantity_name + '_relaxation_timescale'].to_units( - 's') - else: - tau = self._tau.to_units('s') - tendency_unit_string = str( - ureg(state[self._quantity_name].attrs['units']) / ureg('s')) + equilibrium = state['equilibrium_' + self._quantity_name] + tau = state[self._quantity_name + '_relaxation_timescale'] tendencies = { - self._quantity_name: DataArray( - (equilibrium.values - value.values)/tau.values, - dims=value.dims, - attrs={'units': tendency_unit_string} - ) + self._quantity_name: (equilibrium - value)/tau } return tendencies, {} @@ -196,7 +236,10 @@ class TimeDifferencingWrapper(ImplicitPrognostic): >>> component = TimeDifferencingWrapper(GridScaleCondensation()) """ - def __init__(self, implicit): + def __init__(self, implicit, **kwargs): + if len(kwargs) > 0: + raise TypeError('Received unexpected keyword argument {}'.format( + kwargs.popitem()[0])) self._implicit = implicit def __call__(self, state, timestep): @@ -224,6 +267,9 @@ def __call__(self, state, timestep): varname, type(data_array))) return tendencies, diagnostics + def array_call(self, state, timestep): + raise NotImplementedError() + @property def tendencies(self): return list(self.tendency_properties.keys()) diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 9c367ed..9e9721a 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -105,27 +105,27 @@ def __init__( Args ---- - input_scale_factors : dict + input_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and values are floats by which input values are scaled before being used by this object. - output_scale_factors : dict + output_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and values are floats by which output values are scaled before being returned by this object. - diagnostic_scale_factors : dict + diagnostic_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and values are floats by which diagnostic values are scaled before being returned by this object. - tendencies_in_diagnostics : bool + tendencies_in_diagnostics : bool, optional A boolean indicating whether this object will put tendencies of quantities in its diagnostic output based on first order time differencing of output values. - update_interval : timedelta + update_interval : timedelta, optional If given, the component will only give new output if at least a period of update_interval has passed since the last time new output was given. Otherwise, it would return that cached output. - name : string + name : string, optional A label to be used for this object, for example as would be used for Y in the name "X_tendency_from_Y". By default the class name in lowercase is used. @@ -358,19 +358,19 @@ def __init__( Args ---- - input_scale_factors : dict + input_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and values are floats by which input values are scaled before being used by this object. - tendency_scale_factors : dict + tendency_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and values are floats by which tendency values are scaled before being returned by this object. - diagnostic_scale_factors : dict + diagnostic_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and values are floats by which diagnostic values are scaled before being returned by this object. - update_interval : timedelta + update_interval : timedelta, optional If given, the component will only give new output if at least a period of update_interval has passed since the last time new output was given. Otherwise, it would return that cached output. @@ -545,26 +545,26 @@ def __init__( Args ---- - input_scale_factors : dict + input_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and values are floats by which input values are scaled before being used by this object. - tendency_scale_factors : dict + tendency_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and values are floats by which tendency values are scaled before being returned by this object. - diagnostic_scale_factors : dict + diagnostic_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and values are floats by which diagnostic values are scaled before being returned by this object. - tendencies_in_diagnostics : bool + tendencies_in_diagnostics : bool, optional A boolean indicating whether this object will put tendencies of quantities in its diagnostic output. - update_interval : timedelta + update_interval : timedelta, optional If given, the component will only give new output if at least a period of update_interval has passed since the last time new output was given. Otherwise, it would return that cached output. - name : string + name : string, optional A label to be used for this object, for example as would be used for Y in the name "X_tendency_from_Y". By default the class name in lowercase is used. @@ -729,15 +729,15 @@ def __init__( Args ---- - input_scale_factors : dict + input_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and values are floats by which input values are scaled before being used by this object. - diagnostic_scale_factors : dict + diagnostic_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and values are floats by which diagnostic values are scaled before being returned by this object. - update_interval : timedelta + update_interval : timedelta, optional If given, the component will only give new output if at least a period of update_interval has passed since the last time new output was given. Otherwise, it would return that cached output. diff --git a/tests/test_components.py b/tests/test_components.py index b33855d..b8494b0 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -1,13 +1,14 @@ import pytest from sympl import ( - ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, DataArray + ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, DataArray, + timedelta, ) import numpy as np def test_constant_prognostic_empty_dicts(): prog = ConstantPrognostic({}, {}) - tendencies, diagnostics = prog({}) + tendencies, diagnostics = prog({'time': timedelta(0)}) assert isinstance(tendencies, dict) assert isinstance(diagnostics, dict) assert len(tendencies) == 0 @@ -20,24 +21,81 @@ def test_constant_prognostic_cannot_modify_through_input_dict(): prog = ConstantPrognostic(in_tendencies, in_diagnostics) in_tendencies['a'] = 'b' in_diagnostics['c'] = 'd' - tendencies, diagnostics = prog({}) + tendencies, diagnostics = prog({'time': timedelta(0)}) assert len(tendencies) == 0 assert len(diagnostics) == 0 def test_constant_prognostic_cannot_modify_through_output_dict(): prog = ConstantPrognostic({}, {}) - tendencies, diagnostics = prog({}) + tendencies, diagnostics = prog({'time': timedelta(0)}) tendencies['a'] = 'b' diagnostics['c'] = 'd' - tendencies, diagnostics = prog({}) + tendencies, diagnostics = prog({'time': timedelta(0)}) assert len(tendencies) == 0 assert len(diagnostics) == 0 +def test_constant_prognostic_tendency_properties(): + tendencies = { + 'tend1': DataArray( + np.zeros([10]), + dims=['dim1'], + attrs={'units': 'm/s'}, + ), + 'tend2': DataArray( + np.zeros([2, 2]), + dims=['dim2', 'dim3'], + attrs={'units': 'degK/s'}, + ) + } + prog = ConstantPrognostic(tendencies) + assert prog.tendency_properties == { + 'tend1': { + 'dims': ('dim1',), + 'units': 'm/s', + }, + 'tend2': { + 'dims': ('dim2', 'dim3'), + 'units': 'degK/s' + } + } + assert prog.diagnostic_properties == {} + assert prog.input_properties == {} + + +def test_constant_prognostic_diagnostic_properties(): + tendencies = {} + diagnostics = { + 'diag1': DataArray( + np.zeros([10]), + dims=['dim1'], + attrs={'units': 'm'}, + ), + 'diag2': DataArray( + np.zeros([2, 2]), + dims=['dim2', 'dim3'], + attrs={'units': 'degK'}, + ) + } + prog = ConstantPrognostic(tendencies, diagnostics) + assert prog.diagnostic_properties == { + 'diag1': { + 'dims': ('dim1',), + 'units': 'm', + }, + 'diag2': { + 'dims': ('dim2', 'dim3'), + 'units': 'degK', + } + } + assert prog.tendency_properties == {} + assert prog.input_properties == {} + + def test_constant_diagnostic_empty_dict(): diag = ConstantDiagnostic({}) - diagnostics = diag({}) + diagnostics = diag({'time': timedelta(0)}) assert isinstance(diagnostics, dict) assert len(diagnostics) == 0 @@ -46,22 +104,50 @@ def test_constant_diagnostic_cannot_modify_through_input_dict(): in_diagnostics = {} diag = ConstantDiagnostic(in_diagnostics) in_diagnostics['a'] = 'b' - diagnostics = diag({}) + diagnostics = diag({'time': timedelta(0)}) assert isinstance(diagnostics, dict) assert len(diagnostics) == 0 def test_constant_diagnostic_cannot_modify_through_output_dict(): diag = ConstantDiagnostic({}) - diagnostics = diag({}) + diagnostics = diag({'time': timedelta(0)}) diagnostics['c'] = 'd' - diagnostics = diag({}) + diagnostics = diag({'time': timedelta(0)}) assert len(diagnostics) == 0 +def test_constant_diagnostic_diagnostic_properties(): + diagnostics = { + 'diag1': DataArray( + np.zeros([10]), + dims=['dim1'], + attrs={'units': 'm'}, + ), + 'diag2': DataArray( + np.zeros([2, 2]), + dims=['dim2', 'dim3'], + attrs={'units': 'degK'}, + ) + } + diagnostic = ConstantDiagnostic(diagnostics) + assert diagnostic.diagnostic_properties == { + 'diag1': { + 'dims': ('dim1',), + 'units': 'm', + }, + 'diag2': { + 'dims': ('dim2', 'dim3'), + 'units': 'degK', + } + } + assert diagnostic.input_properties == {} + + def test_relaxation_prognostic_at_equilibrium(): - prognostic = RelaxationPrognostic('quantity') + prognostic = RelaxationPrognostic('quantity', 'degK') state = { + 'time': timedelta(0), 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'degK'}), 'quantity_relaxation_timescale': DataArray( np.array([1., 1., 1.]), attrs={'units': 's'}), @@ -73,8 +159,9 @@ def test_relaxation_prognostic_at_equilibrium(): def test_relaxation_prognostic_with_change(): - prognostic = RelaxationPrognostic('quantity') + prognostic = RelaxationPrognostic('quantity', 'degK') state = { + 'time': timedelta(0), 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'degK'}), 'quantity_relaxation_timescale': DataArray( np.array([1., 1., 1.]), attrs={'units': 's'}), @@ -86,8 +173,9 @@ def test_relaxation_prognostic_with_change(): def test_relaxation_prognostic_with_change_different_timescale_units(): - prognostic = RelaxationPrognostic('quantity') + prognostic = RelaxationPrognostic('quantity', 'degK') state = { + 'time': timedelta(0), 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'degK'}), 'quantity_relaxation_timescale': DataArray( np.array([1/60., 2/60., 3/60.]), attrs={'units': 'minutes'}), @@ -99,8 +187,9 @@ def test_relaxation_prognostic_with_change_different_timescale_units(): def test_relaxation_prognostic_with_change_different_equilibrium_units(): - prognostic = RelaxationPrognostic('quantity') + prognostic = RelaxationPrognostic('quantity', 'm') state = { + 'time': timedelta(0), 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'm'}), 'quantity_relaxation_timescale': DataArray( np.array([1., 2., 3.]), attrs={'units': 's'}), @@ -111,37 +200,5 @@ def test_relaxation_prognostic_with_change_different_equilibrium_units(): assert np.all(tendencies['quantity'].values == np.array([1., 1., 1.])) -def test_relaxation_prognostic_caching_timescale_and_equilibrium(): - prognostic = RelaxationPrognostic( - 'quantity', - relaxation_timescale=DataArray( - np.array([1., 1., 1.]), attrs={'units': 's'}), - equilibrium_value=DataArray( - np.array([1., 3., 5.]), attrs={'units': 'degK'})) - state = { - 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'degK'}), - } - tendencies, diagnostics = prognostic(state) - assert np.all(tendencies['quantity'].values == np.array([1., 2., 3.])) - - -def test_relaxation_prognostic_cannot_override_cached_values(): - prognostic = RelaxationPrognostic( - 'quantity', - relaxation_timescale=DataArray( - np.array([1., 1., 1.]), attrs={'units': 's'}), - equilibrium_value=DataArray( - np.array([1., 3., 5.]), attrs={'units': 'degK'})) - state = { - 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'degK'}), - # random values which should not be used by the prognostic - 'quantity_relaxation_timescale': DataArray( - np.array([5., 1., 18.]), attrs={'units': 's'}), - 'equilibrium_quantity': DataArray( - np.array([4., 7., 2.])*1e-3, attrs={'units': 'km'}), - } - tendencies, diagnostics = prognostic(state) - assert np.all(tendencies['quantity'].values == np.array([1., 2., 3.])) - if __name__ == '__main__': pytest.main([__file__]) From 244bcb260f09d4f333297e3240d9499e03d811f1 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 23 Mar 2018 15:42:30 -0700 Subject: [PATCH 21/98] Components now enforce consistency of array_call output with properties dicts --- HISTORY.rst | 3 + sympl/__init__.py | 6 +- sympl/_core/base_components.py | 167 +++++++++++++++++++++++++++++-- sympl/_core/exceptions.py | 8 ++ tests/test_base_components.py | 174 ++++++++++++++++++++++++++++++++- 5 files changed, 349 insertions(+), 9 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 22a500b..a3922c8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -24,6 +24,9 @@ Latest is done using first-order time differencing. * Composites now have properties dictionaries. * Updated basic components to use new component API. +* Components enforce consistency of output from array_call with properties + dictionaries, raising ComponentMissingOutputError or ComponentExtraOutputError + respectively if outputs do not match. Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/__init__.py b/sympl/__init__.py index 51a532c..ee9c2ba 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -8,7 +8,8 @@ from ._components.timesteppers import AdamsBashforth, Leapfrog, SSPRungeKutta from ._core.exceptions import ( InvalidStateError, SharedKeyError, DependencyError, - InvalidPropertyDictError) + InvalidPropertyDictError, ComponentExtraOutputError, + ComponentMissingOutputError) from ._core.array import DataArray from ._core.constants import ( get_constant, set_constant, set_condensible_name, reset_constants, @@ -32,7 +33,8 @@ DiagnosticComposite, MonitorComposite, ImplicitPrognostic, TimeStepper, Leapfrog, AdamsBashforth, SSPRungeKutta, InvalidStateError, SharedKeyError, DependencyError, - InvalidPropertyDictError, + InvalidPropertyDictError, ComponentExtraOutputError, + ComponentMissingOutputError, DataArray, get_constant, set_constant, set_condensible_name, reset_constants, get_constants_string, TimeDifferencingWrapper, diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 9e9721a..8f179d2 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -2,7 +2,9 @@ from .util import ( get_numpy_arrays_with_properties, restore_data_arrays_with_properties) from .time import timedelta -from .exceptions import InvalidPropertyDictError +from .exceptions import ( + InvalidPropertyDictError, ComponentExtraOutputError, + ComponentMissingOutputError) def apply_scale_factors(array_state, scale_factors): @@ -10,7 +12,135 @@ def apply_scale_factors(array_state, scale_factors): array_state[key] *= factor -class Implicit(object): +class TendencyMixin(object): + + @property + def _wanted_tendency_aliases(self): + wanted_tendency_aliases = {} + for name, properties in self.tendency_properties.items(): + wanted_tendency_aliases[name] = [] + if 'alias' in properties.keys(): + wanted_tendency_aliases[name].append(properties['alias']) + if (name in self.input_properties.keys() and + 'alias' in self.input_properties[name].keys()): + wanted_tendency_aliases[name].append(self.input_properties[name]['alias']) + return wanted_tendency_aliases + + def _check_missing_tendencies(self, tendency_dict): + missing_tendencies = set() + for name, aliases in self._wanted_tendency_aliases.items(): + if (name not in tendency_dict.keys() and + not any(alias in tendency_dict.keys() for alias in aliases)): + missing_tendencies.add(name) + if len(missing_tendencies) > 0: + raise ComponentMissingOutputError( + 'Component {} did not compute tendencies for {}'.format( + self.__class__.__name__, ', '.join(missing_tendencies))) + + def _check_extra_tendencies(self, tendency_dict): + wanted_set = set() + wanted_set.update(self._wanted_tendency_aliases.keys()) + for value_list in self._wanted_tendency_aliases.values(): + wanted_set.update(value_list) + extra_tendencies = set(tendency_dict.keys()).difference(wanted_set) + if len(extra_tendencies) > 0: + raise ComponentExtraOutputError( + 'Component {} computed tendencies for {} which are not in ' + 'tendency_properties'.format( + self.__class__.__name__, ', '.join(extra_tendencies))) + + def _check_tendencies(self, tendency_dict): + self._check_missing_tendencies(tendency_dict) + self._check_extra_tendencies(tendency_dict) + + +class DiagnosticMixin(object): + + @property + def _wanted_diagnostic_aliases(self): + wanted_diagnostic_aliases = {} + for name, properties in self.diagnostic_properties.items(): + wanted_diagnostic_aliases[name] = [] + if 'alias' in properties.keys(): + wanted_diagnostic_aliases[name].append(properties['alias']) + if (name in self.input_properties.keys() and + 'alias' in self.input_properties[name].keys()): + wanted_diagnostic_aliases[name].append(self.input_properties[name]['alias']) + return wanted_diagnostic_aliases + + def _check_missing_diagnostics(self, diagnostics_dict): + missing_diagnostics = set() + for name, aliases in self._wanted_diagnostic_aliases.items(): + if (name not in diagnostics_dict.keys() and + not any(alias in diagnostics_dict.keys() for alias in aliases)): + missing_diagnostics.add(name) + if len(missing_diagnostics) > 0: + raise ComponentMissingOutputError( + 'Component {} did not compute diagnostics {}'.format( + self.__class__.__name__, ', '.join(missing_diagnostics))) + + def _check_extra_diagnostics(self, diagnostics_dict): + wanted_set = set() + wanted_set.update(self._wanted_diagnostic_aliases.keys()) + for value_list in self._wanted_diagnostic_aliases.values(): + wanted_set.update(value_list) + extra_diagnostics = set(diagnostics_dict.keys()).difference(wanted_set) + if len(extra_diagnostics) > 0: + raise ComponentExtraOutputError( + 'Component {} computed diagnostics {} which are not in ' + 'diagnostic_properties'.format( + self.__class__.__name__, ', '.join(extra_diagnostics))) + + def _check_diagnostics(self, diagnostics_dict): + self._check_missing_diagnostics(diagnostics_dict) + self._check_extra_diagnostics(diagnostics_dict) + + +class OutputMixin(object): + + @property + def _wanted_output_aliases(self): + wanted_output_aliases = {} + for name, properties in self.output_properties.items(): + wanted_output_aliases[name] = [] + if 'alias' in properties.keys(): + wanted_output_aliases[name].append(properties['alias']) + if (name in self.input_properties.keys() and + 'alias' in self.input_properties[name].keys()): + wanted_output_aliases[name].append( + self.input_properties[name]['alias']) + return wanted_output_aliases + + def _check_missing_outputs(self, outputs_dict): + missing_outputs = set() + for name, aliases in self._wanted_output_aliases.items(): + if (name not in outputs_dict.keys() and + not any(alias in outputs_dict.keys() for alias in + aliases)): + missing_outputs.add(name) + if len(missing_outputs) > 0: + raise ComponentMissingOutputError( + 'Component {} did not compute outputs {}'.format( + self.__class__.__name__, ', '.join(missing_outputs))) + + def _check_extra_outputs(self, outputs_dict): + wanted_set = set() + wanted_set.update(self._wanted_output_aliases.keys()) + for value_list in self._wanted_output_aliases.values(): + wanted_set.update(value_list) + extra_outputs = set(outputs_dict.keys()).difference(wanted_set) + if len(extra_outputs) > 0: + raise ComponentExtraOutputError( + 'Component {} computed outputs {} which are not in ' + 'output_properties'.format( + self.__class__.__name__, ', '.join(extra_outputs))) + + def _check_outputs(self, output_dict): + self._check_missing_outputs(output_dict) + self._check_extra_outputs(output_dict) + + +class Implicit(DiagnosticMixin, OutputMixin): """ Attributes ---------- @@ -150,9 +280,12 @@ def __init__( else: self.name = name if tendencies_in_diagnostics: - self._insert_tendency_properties() + self._added_tendency_properties = self._insert_tendency_properties() + else: + self._added_tendency_properties = set() def _insert_tendency_properties(self): + added_names = [] for name, properties in self.output_properties.items(): tendency_name = self._get_tendency_name(name) if properties['units'] is '': @@ -186,6 +319,20 @@ def _insert_tendency_properties(self): self.output_properties[name]['units'] ) ) + added_names.append(tendency_name) + return added_names + + def _check_missing_diagnostics(self, diagnostics_dict): + missing_diagnostics = set() + for name, aliases in self._wanted_diagnostic_aliases.items(): + if (name not in diagnostics_dict.keys() and + name not in self._added_tendency_properties and + not any(alias in diagnostics_dict.keys() for alias in aliases)): + missing_diagnostics.add(name) + if len(missing_diagnostics) > 0: + raise ComponentMissingOutputError( + 'Component {} did not compute diagnostics {}'.format( + self.__class__.__name__, ', '.join(missing_diagnostics))) def _get_tendency_name(self, name): return '{}_tendency_from_{}'.format(name, self.name) @@ -230,6 +377,8 @@ def __call__(self, state, timestep): raw_state['time'] = state['time'] apply_scale_factors(raw_state, self.input_scale_factors) raw_diagnostics, raw_new_state = self.array_call(raw_state, timestep) + self._check_diagnostics(raw_diagnostics) + self._check_outputs(raw_new_state) apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) apply_scale_factors(raw_new_state, self.output_scale_factors) if self.tendencies_in_diagnostics: @@ -279,7 +428,7 @@ def array_call(self, state, timestep): pass -class Prognostic(object): +class Prognostic(DiagnosticMixin, TendencyMixin): """ Attributes ---------- @@ -425,6 +574,8 @@ def __call__(self, state): raw_state['time'] = state['time'] apply_scale_factors(raw_state, self.input_scale_factors) raw_tendencies, raw_diagnostics = self.array_call(raw_state) + self._check_tendencies(raw_tendencies) + self._check_diagnostics(raw_diagnostics) apply_scale_factors(raw_tendencies, self.tendency_scale_factors) apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) self._tendencies = restore_data_arrays_with_properties( @@ -436,6 +587,7 @@ def __call__(self, state): self._last_update_time = state['time'] return self._tendencies, self._diagnostics + @abc.abstractmethod def array_call(self, state): """ @@ -464,7 +616,7 @@ def array_call(self, state): pass -class ImplicitPrognostic(object): +class ImplicitPrognostic(DiagnosticMixin, TendencyMixin): """ Attributes ---------- @@ -625,6 +777,8 @@ def __call__(self, state, timestep): raw_state['time'] = state['time'] apply_scale_factors(raw_state, self.input_scale_factors) raw_tendencies, raw_diagnostics = self.array_call(raw_state, timestep) + self._check_tendencies(raw_tendencies) + self._check_diagnostics(raw_diagnostics) apply_scale_factors(raw_tendencies, self.tendency_scale_factors) apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) self._tendencies = restore_data_arrays_with_properties( @@ -665,7 +819,7 @@ def array_call(self, state, timestep): """ -class Diagnostic(object): +class Diagnostic(DiagnosticMixin): """ Attributes ---------- @@ -784,6 +938,7 @@ def __call__(self, state): raw_state['time'] = state['time'] apply_scale_factors(raw_state, self.input_scale_factors) raw_diagnostics = self.array_call(raw_state) + self._check_diagnostics(raw_diagnostics) apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) self._diagnostics = restore_data_arrays_with_properties( raw_diagnostics, self.diagnostic_properties, diff --git a/sympl/_core/exceptions.py b/sympl/_core/exceptions.py index 8c8b0e0..4c71df8 100644 --- a/sympl/_core/exceptions.py +++ b/sympl/_core/exceptions.py @@ -12,3 +12,11 @@ class SharedKeyError(Exception): class DependencyError(Exception): pass + + +class ComponentMissingOutputError(Exception): + pass + + +class ComponentExtraOutputError(Exception): + pass diff --git a/tests/test_base_components.py b/tests/test_base_components.py index 561c9a9..26f2eee 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -4,7 +4,8 @@ import unittest from sympl import ( Prognostic, Diagnostic, Monitor, Implicit, ImplicitPrognostic, - datetime, timedelta, DataArray, InvalidPropertyDictError + datetime, timedelta, DataArray, InvalidPropertyDictError, + ComponentMissingOutputError, ComponentExtraOutputError, ) def same_list(list1, list2): @@ -134,6 +135,76 @@ def test_empty_prognostic(self): assert prognostic.state_given['time'] == timedelta(seconds=0) assert prognostic.times_called == 1 + def test_raises_when_tendency_not_given(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'tend1': { + 'dims': ['dims1'], + 'units': 'm', + } + } + diagnostic_output = {} + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentMissingOutputError): + _, _ = self.call_component(prognostic, state) + + def test_raises_when_extraneous_tendency_given(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = { + 'tend1': np.zeros([10]), + } + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentExtraOutputError): + _, _ = self.call_component(prognostic, state) + + def test_raises_when_diagnostic_not_given(self): + input_properties = {} + diagnostic_properties = { + 'diag1': { + 'dims': ['dims1'], + 'units': 'm', + } + } + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentMissingOutputError): + _, _ = self.call_component(prognostic, state) + + def test_raises_when_extraneous_diagnostic_given(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = { + 'diag1': np.zeros([10]) + } + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentExtraOutputError): + _, _ = self.call_component(prognostic, state) + def test_input_no_transformations(self): input_properties = { 'input1': { @@ -737,6 +808,37 @@ def test_empty_diagnostic(self): assert diagnostic.state_given['time'] == timedelta(seconds=0) assert diagnostic.times_called == 1 + def test_raises_when_diagnostic_not_given(self): + input_properties = {} + diagnostic_properties = { + 'diag1': { + 'dims': ['dims1'], + 'units': 'm', + } + } + diagnostic_output = {} + diagnostic = self.component_class( + input_properties, diagnostic_properties, + diagnostic_output, + ) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentMissingOutputError): + _, _ = self.call_component(diagnostic, state) + + def test_raises_when_extraneous_diagnostic_given(self): + input_properties = {} + diagnostic_properties = {} + diagnostic_output = { + 'diag1': np.zeros([10]) + } + diagnostic = self.component_class( + input_properties, diagnostic_properties, + diagnostic_output, + ) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentExtraOutputError): + _, _ = self.call_component(diagnostic, state) + def test_input_no_transformations(self): input_properties = { 'input1': { @@ -1139,6 +1241,76 @@ def test_timedelta_is_passed(self): assert implicit.timestep_given == timedelta(seconds=5) assert implicit.times_called == 1 + def test_raises_when_output_not_given(self): + input_properties = {} + diagnostic_properties = {} + output_properties = { + 'output1': { + 'dims': ['dims1'], + 'units': 'm', + } + } + diagnostic_output = {} + state_output = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, state_output + ) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentMissingOutputError): + _, _ = self.call_component(implicit, state) + + def test_raises_when_extraneous_output_given(self): + input_properties = {} + diagnostic_properties = {} + output_properties = {} + diagnostic_output = {} + state_output = { + 'tend1': np.zeros([10]), + } + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, state_output + ) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentExtraOutputError): + _, _ = self.call_component(implicit, state) + + def test_raises_when_diagnostic_not_given(self): + input_properties = {} + diagnostic_properties = { + 'diag1': { + 'dims': ['dims1'], + 'units': 'm', + } + } + output_properties = {} + diagnostic_output = {} + state_output = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, state_output + ) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentMissingOutputError): + _, _ = self.call_component(implicit, state) + + def test_raises_when_extraneous_diagnostic_given(self): + input_properties = {} + diagnostic_properties = {} + output_properties = {} + diagnostic_output = { + 'diag1': np.zeros([10]) + } + state_output = {} + implicit = self.component_class( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, state_output + ) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentExtraOutputError): + _, _ = self.call_component(implicit, state) + def test_input_no_transformations(self): input_properties = { 'input1': { From 0d0c3a974c73729ba2d30ffd7382001e5381d742 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 23 Mar 2018 15:43:39 -0700 Subject: [PATCH 22/98] flake8 --- sympl/_components/basic.py | 1 - sympl/_core/base_components.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sympl/_components/basic.py b/sympl/_components/basic.py index de697f2..874db61 100644 --- a/sympl/_components/basic.py +++ b/sympl/_components/basic.py @@ -1,7 +1,6 @@ from .._core.array import DataArray from .._core.base_components import ImplicitPrognostic, Prognostic, Diagnostic from .._core.units import unit_registry as ureg -from .._core.util import combine_dims class ConstantPrognostic(Prognostic): diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 8f179d2..d9afa0a 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -106,7 +106,7 @@ def _wanted_output_aliases(self): if 'alias' in properties.keys(): wanted_output_aliases[name].append(properties['alias']) if (name in self.input_properties.keys() and - 'alias' in self.input_properties[name].keys()): + 'alias' in self.input_properties[name].keys()): wanted_output_aliases[name].append( self.input_properties[name]['alias']) return wanted_output_aliases @@ -587,7 +587,6 @@ def __call__(self, state): self._last_update_time = state['time'] return self._tendencies, self._diagnostics - @abc.abstractmethod def array_call(self, state): """ From 32e07be8edc5788c7a813033cb4a4118b1055a2b Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 23 Mar 2018 15:53:06 -0700 Subject: [PATCH 23/98] Removed ComponentTestBase from API --- HISTORY.rst | 2 + sympl/__init__.py | 2 - sympl/_core/testing.py | 310 ----------------------------------------- 3 files changed, 2 insertions(+), 312 deletions(-) delete mode 100644 sympl/_core/testing.py diff --git a/HISTORY.rst b/HISTORY.rst index a3922c8..fddf1e4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -55,6 +55,8 @@ Breaking changes * RelaxationPrognostic no longer allows caching of equilibrium values or timescale. They must be provided through the input state. This is to ensure proper conversion of dimensions and units. +* Removed ComponentTestBase from package. All of its tests except for output + caching are now performed on object initialization or call time. v0.3.2 ------ diff --git a/sympl/__init__.py b/sympl/__init__.py index ee9c2ba..bb69dc1 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -20,7 +20,6 @@ restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, get_component_aliases) -from ._core.testing import ComponentTestBase from ._components import ( PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, @@ -43,7 +42,6 @@ restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, get_component_aliases, - ComponentTestBase, PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, datetime, timedelta diff --git a/sympl/_core/testing.py b/sympl/_core/testing.py deleted file mode 100644 index f2b3091..0000000 --- a/sympl/_core/testing.py +++ /dev/null @@ -1,310 +0,0 @@ -import abc -import os -from glob import glob -import xarray as xr -from .util import same_list -from .base_components import Diagnostic, Prognostic, Implicit -from sympl._core.timestepper import TimeStepper -import numpy as np -from .units import is_valid_unit -from datetime import timedelta - - -def cache_dictionary(dictionary, filename): - dataset = xr.Dataset(dictionary) - dataset.to_netcdf(filename, engine='scipy') - - -def load_dictionary(filename): - dataset = xr.open_dataset(filename, engine='scipy') - return dict(dataset.data_vars) - - -def call_with_timestep_if_needed( - component, state, timestep=timedelta(seconds=10.)): - if isinstance(component, (Implicit, TimeStepper)): - return component(state, timestep=timestep) - else: - return component(state) - - -def compare_outputs(current, cached): - if isinstance(current, tuple) and isinstance(cached, tuple): - for i in range(len(current)): - compare_one_state_pair(current[i], cached[i]) - elif (not isinstance(current, tuple)) and (not isinstance(cached, tuple)): - compare_one_state_pair(current, cached) - else: - raise AssertionError('Different number of dicts returned than cached.') - - -def compare_one_state_pair(current, cached): - for key in current.keys(): - try: - assert np.all(np.isclose(current[key].values, cached[key].values)) - for attr in current[key].attrs: - assert current[key].attrs[attr] == cached[key].attrs[attr] - for attr in cached[key].attrs: - assert attr in current[key].attrs - assert current[key].dims == cached[key].dims - except AssertionError as err: - raise AssertionError('Error for {}: {}'.format(key, err)) - for key in cached.keys(): - assert key in current.keys() - - -def assert_dimension_lengths_are_consistent(state): - dimension_lengths = {} - for name, value in state.items(): - if name != 'time': - for i, dim_name in enumerate(value.dims): - try: - if dim_name in dimension_lengths: - assert dimension_lengths[dim_name] == value.shape[i] - else: - dimension_lengths[dim_name] = value.shape[i] - except AssertionError as err: - raise AssertionError( - 'Inconsistent length on dimension {} for value {}:' - '{}'.format(dim_name, name, err)) - - -class ComponentTestBase(object): - - cache_folder = None - - @abc.abstractmethod - def get_input_state(self): - pass - - @abc.abstractmethod - def get_component_instance(self): - pass - - def get_cached_output(self,): - cache_filename_list = sorted(glob( - os.path.join( - self.cache_folder, - '{}-*.cache'.format( - self.__class__.__name__)))) - if len(cache_filename_list) > 0: - return_list = [] - for filename in cache_filename_list: - return_list.append(load_dictionary(filename)) - if len(return_list) > 1: - return tuple(return_list) - elif len(return_list) == 1: - return return_list[0] - else: - return None - - def cache_output(self, output): - if not isinstance(output, tuple): - output = (output,) - for i in range(len(output)): - cache_filename = os.path.join( - self.cache_folder, '{}-{}.cache'.format(self.__class__.__name__, i)) - cache_dictionary(output[i], cache_filename) - - def test_output_matches_cached_output(self): - state = self.get_input_state() - component = self.get_component_instance() - output = call_with_timestep_if_needed(component, state) - cached_output = self.get_cached_output() - if cached_output is None: - self.cache_output(output) - raise AssertionError( - 'Failed due to no cached output, cached current output') - else: - compare_outputs(output, cached_output) - - def test_component_listed_inputs_are_accurate(self): - state = self.get_input_state() - component = self.get_component_instance() - input_state = {} - for key in component.inputs: - input_state[key] = state[key] - output = call_with_timestep_if_needed(component, state) - cached_output = self.get_cached_output() - if cached_output is not None: - compare_outputs(output, cached_output) - - def test_inputs_and_outputs_have_consistent_dim_lengths(self): - """A given dimension name should always have the same length.""" - input_state = self.get_input_state() - assert_dimension_lengths_are_consistent(input_state) - component = self.get_component_instance() - output = call_with_timestep_if_needed(component, input_state) - if isinstance(output, tuple): - # Check diagnostics/tendencies/outputs are consistent with one - # another - test_state = {} - for state in output: - test_state.update(state) - assert_dimension_lengths_are_consistent(test_state) - else: - test_state = output # if not a tuple assume it's a dict - assert_dimension_lengths_are_consistent(test_state) - - def test_listed_outputs_are_accurate(self): - state = self.get_input_state() - component = self.get_component_instance() - if isinstance(component, Diagnostic): - diagnostics = component(state) - assert same_list(component.diagnostic_properties.keys(), diagnostics.keys()) - elif isinstance(component, Prognostic): - tendencies, diagnostics = component(state) - assert same_list(component.tendency_properties.keys(), tendencies.keys()) - assert same_list(component.diagnostic_properties.keys(), diagnostics.keys()) - elif isinstance(component, Implicit): - diagnostics, new_state = component(state) - assert same_list(component.diagnostic_properties.keys(), diagnostics.keys()) - assert same_list(component.output_properties.keys(), new_state.keys()) - - def test_modifies_attribute_is_accurate(self): - state = self.get_input_state() - component = self.get_component_instance() - if not hasattr(component, 'modifies'): - raise AssertionError("component does not have a 'modifies' property") - original_state = {} - for key, value in state.items(): - if key == 'time': - original_state[key] = state[key] - else: - original_state[key] = state[key].copy(deep=True) - component(state) - for key in state.keys(): - if key not in state.modifies: - assert np.all(original_state[key] == state[key]), key - - def test_has_input_properties(self): - component = self.get_component_instance() - assert hasattr(component, 'input_properties') - assert isinstance(component.input_properties, dict) - - def test_has_output_properties(self): - component = self.get_component_instance() - if isinstance(component, [Implicit, TimeStepper]): - assert hasattr(component, 'output_properties') - assert isinstance(component.output_properties, dict) - - def test_has_tendency_properties(self): - component = self.get_component_instance() - if isinstance(component, Prognostic): - assert hasattr(component, 'tendency_properties') - assert isinstance(component.tendency_properties, dict) - - def test_has_diagnostic_properties(self): - component = self.get_component_instance() - if isinstance(component, [Diagnostic, Prognostic, Implicit, TimeStepper]): - assert hasattr(component, 'diagnostic_properties') - assert isinstance(component.diagnostic_properties, dict) - - def test_input_unit_properties_are_valid(self): - component = self.get_component_instance() - for name, properties in component.input_properties.items(): - if 'units' not in properties.keys(): - raise AssertionError( - "quantity {} has no 'units' property defined".format(name)) - if not is_valid_unit(properties['units']): - raise AssertionError( - "unit {} for quantity {} is not recognized".format( - properties['units'], name - ) - ) - - def test_diagnostic_unit_properties_are_valid(self): - component = self.get_component_instance() - if hasattr(component, 'diagnostic_properties'): - for name, properties in component.diagnostic_properties.items(): - if 'units' not in properties.keys(): - raise AssertionError( - "quantity {} has no 'units' property defined in " - "diagnostic_properties".format(name)) - if not is_valid_unit(properties['units']): - raise AssertionError( - "unit {} for quantity {} in diagnostic_properties " - "is not recognized".format( - properties['units'], name - ) - ) - - def test_tendency_unit_properties_are_valid(self): - component = self.get_component_instance() - if hasattr(component, 'tendency_properties'): - for name, properties in component.tendency_properties.items(): - if 'units' not in properties.keys(): - raise AssertionError( - "quantity {} has no 'units' property defined in " - "tendency_properties".format(name)) - if not is_valid_unit(properties['units']): - raise AssertionError( - "unit {} for quantity {} in tendency_properties " - "is not recognized".format( - properties['units'], name - ) - ) - - def test_output_unit_properties_are_valid(self): - component = self.get_component_instance() - if hasattr(component, 'output_properties'): - for name, properties in component.output_properties.items(): - if 'units' not in properties.keys(): - raise AssertionError( - "quantity {} has no 'units' property defined in " - "output_properties".format(name)) - if not is_valid_unit(properties['units']): - raise AssertionError( - "unit {} for quantity {} in output_properties " - "is not recognized".format( - properties['units'], name - ) - ) - - def test_tendency_dims_like_properties_are_inputs(self): - component = self.get_component_instance() - if hasattr(component, 'tendency_properties'): - for name, properties in component.tendency_properties.items(): - if 'dims_like' not in properties.keys(): - raise AssertionError( - "quantity {} has no 'dims_like' property defined in " - "tendency_properties".format(name)) - if properties['dims_like'] not in component.inputs: - raise AssertionError( - 'quantity {} has dims_like {} in tendency_properties, ' - 'but {} is not specified as an input'.format( - name, properties['dims_like'], - properties['dims_like']) - ) - - def test_diagnostic_dims_like_properties_are_inputs(self): - component = self.get_component_instance() - if hasattr(component, 'diagnostic_properties'): - for name, properties in component.diagnostic_properties.items(): - if 'dims_like' not in properties.keys(): - raise AssertionError( - "quantity {} has no 'dims_like' property defined in " - "diagnostic_properties".format(name)) - if properties['dims_like'] not in component.inputs: - raise AssertionError( - 'quantity {} has dims_like {} in diagnostic_properties, ' - 'but {} is not specified as an input'.format( - name, properties['dims_like'], - properties['dims_like']) - ) - - def test_output_dims_like_properties_are_inputs(self): - component = self.get_component_instance() - if hasattr(component, 'output_properties'): - for name, properties in component.output_properties.items(): - if 'dims_like' not in properties.keys(): - raise AssertionError( - "quantity {} has no 'dims_like' property defined in " - "output_properties".format(name)) - if properties['dims_like'] not in component.inputs: - raise AssertionError( - 'quantity {} has dims_like {} in output_properties, ' - 'but {} is not specified as an input'.format( - name, properties['dims_like'], - properties['dims_like']) - ) From 391bbee43aad1e0b299f8ebac698739e490164e4 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 23 Mar 2018 16:49:00 -0700 Subject: [PATCH 24/98] More test writing and code fixes --- HISTORY.rst | 2 + sympl/_components/basic.py | 24 +- sympl/_core/composite.py | 40 ++- sympl/_core/util.py | 40 +-- tests/test_time_differencing_wrapper.py | 16 +- tests/test_util.py | 427 +++++++++++++++++++----- 6 files changed, 420 insertions(+), 129 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fddf1e4..2f5bf53 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,8 @@ Latest * Components enforce consistency of output from array_call with properties dictionaries, raising ComponentMissingOutputError or ComponentExtraOutputError respectively if outputs do not match. +* Added a priority order of property types for determining which aliases are + returned by get_component_aliases Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/_components/basic.py b/sympl/_components/basic.py index 874db61..da6af52 100644 --- a/sympl/_components/basic.py +++ b/sympl/_components/basic.py @@ -235,6 +235,20 @@ class TimeDifferencingWrapper(ImplicitPrognostic): >>> component = TimeDifferencingWrapper(GridScaleCondensation()) """ + @property + def input_properties(self): + return self._implicit.input_properties + + @property + def tendency_properties(self): + return_dict = self._implicit.output_properties.copy() + return_dict.update(self._tendency_diagnostic_properties) + return return_dict + + @property + def diagnostic_properties(self): + return self._implicit.diagnostic_propertes + def __init__(self, implicit, **kwargs): if len(kwargs) > 0: raise TypeError('Received unexpected keyword argument {}'.format( @@ -269,16 +283,6 @@ def __call__(self, state, timestep): def array_call(self, state, timestep): raise NotImplementedError() - @property - def tendencies(self): - return list(self.tendency_properties.keys()) - - @property - def tendency_properties(self): - return_dict = self._implicit.output_properties.copy() - return_dict.update(self._tendency_diagnostic_properties) - return return_dict - def __getattr__(self, item): if item not in ('outputs', 'output_properties'): return getattr(self._implicit, item) diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 85c3386..1eafcf1 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -17,17 +17,6 @@ class ComponentComposite(object): component_class = None - @property - def input_properties(self): - return combine_component_properties(self.component_list, 'input_properties') - - @property - def diagnostic_properties(self): - return_dict = {} - for component in self.component_list: - return_dict.update(component.diagnostic_properties) - return return_dict - def __str__(self): return '{}(\n{}\n)'.format( self.__class__, @@ -53,6 +42,7 @@ def __init__(self, *args): if self.component_class is not None: ensure_components_have_class(args, self.component_class) self.component_list = args + super(ComponentComposite, self).__init__() def ensure_no_diagnostic_output_overlap(self): diagnostic_names = [] @@ -84,18 +74,28 @@ class PrognosticComposite(ComponentComposite, Prognostic): component_class = Prognostic + @property + def input_properties(self): + return combine_component_properties(self.component_list, 'input_properties') + + @property + def diagnostic_properties(self): + return_dict = {} + for component in self.component_list: + return_dict.update(component.diagnostic_properties) + return return_dict + @property def tendency_properties(self): return combine_component_properties(self.component_list, 'tendency_properties') def __init__(self, *args): super(PrognosticComposite, self).__init__(*args) - self.ensure_tendency_outputs_are_compatible() self.ensure_no_diagnostic_output_overlap() - - def ensure_tendency_outputs_are_compatible(self): + self.input_properties self.tendency_properties + def __call__(self, state): """ Gets tendencies and diagnostics from the passed model state. @@ -142,9 +142,21 @@ class DiagnosticComposite(ComponentComposite, Diagnostic): component_class = Diagnostic + @property + def input_properties(self): + return combine_component_properties(self.component_list, 'input_properties') + + @property + def diagnostic_properties(self): + return_dict = {} + for component in self.component_list: + return_dict.update(component.diagnostic_properties) + return return_dict + def __init__(self, *args): super(DiagnosticComposite, self).__init__(*args) self.ensure_no_diagnostic_output_overlap() + self.input_properties def __call__(self, state): """ diff --git a/sympl/_core/util.py b/sympl/_core/util.py index 24005ef..42b6aad 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -671,13 +671,13 @@ def combine_properties(*args): def get_component_aliases(*args): """ - Returns aliases for variables in the properties of Components (e.g., Prognostics). + Returns aliases for variables in the properties of Components (Prognostic, + Diagnostic, Implicit, and ImplicitPrognostic objects). - Notes - ----- - - If a variable shows up in the input_properties or diagnostic_properties - of two or more different Components, make sure they have the same 'alias' - keyword in all Components. + If multiple aliases are present for the same variable, the following + properties have priority in descending order: input, output, diagnostic, + tendency. If multiple components give different aliases at the same priority + level, one is chosen arbitrarily. Args ---- @@ -688,22 +688,16 @@ def get_component_aliases(*args): Returns ------- aliases : dict - A dictionary with keys containing old variable names and values containing - new variable names + A dictionary mapping quantity names to aliases """ - - aliases = {} - - # Update the aliases dict with the properties in each provided Component - for component in args: - # combine the input, output, diagnostic, and tendency variables into one dict - for prop_type in ['input_properties', 'output_properties', - 'diagnostic_properties', 'tendency_properties']: - if hasattr(component, prop_type): - component_properties = getattr(component, prop_type) - # save the alias (if there is one) for each variable - for varname, properties in component_properties.items(): + return_dict = {} + for property_type in ( + 'tendency_properties', 'diagnostic_properties', 'output_properties', + 'input_properties'): + for component in args: + if hasattr(component, property_type): + component_properties = getattr(component, property_type) + for name, properties in component_properties.items(): if 'alias' in properties.keys(): - aliases.update({varname: properties['alias']}) - - return aliases + return_dict[name] = properties['alias'] + return return_dict diff --git a/tests/test_time_differencing_wrapper.py b/tests/test_time_differencing_wrapper.py index cb36f83..8e6873b 100644 --- a/tests/test_time_differencing_wrapper.py +++ b/tests/test_time_differencing_wrapper.py @@ -20,6 +20,8 @@ def __call__(self, state): class MockImplicit(Implicit): + input_properties = {} + output_properties = { 'value': { 'dims': [], @@ -36,13 +38,15 @@ class MockImplicit(Implicit): def __init__(self): self._num_updates = 0 + super(MockImplicit, self).__init__() - def __call__(self, state, timestep): + def array_call(self, state, timestep): self._num_updates += 1 return ( - {'num_updates': DataArray([self._num_updates], attrs={'units': ''})}, - {'value': DataArray([1], attrs={'units': 'm'})}) + {'num_updates': self._num_updates}, + {'value': 1} + ) class MockImplicitThatExpects(Implicit): @@ -107,6 +111,7 @@ def setUp(self): self.implicit = MockImplicit() self.wrapped = TimeDifferencingWrapper(self.implicit) self.state = { + 'time': timedelta(0), 'value': DataArray([0], attrs={'units': 'm'}) } @@ -115,9 +120,9 @@ def tearDown(self): def testWrapperCallsImplicit(self): tendencies, diagnostics = self.wrapped(self.state, timedelta(seconds=1)) - assert diagnostics['num_updates'].values[0] == 1 + assert diagnostics['num_updates'].values == 1 tendencies, diagnostics = self.wrapped(self.state, timedelta(seconds=1)) - assert diagnostics['num_updates'].values[0] == 2 + assert diagnostics['num_updates'].values == 2 assert len(diagnostics.keys()) == 1 def testWrapperComputesTendency(self): @@ -130,6 +135,7 @@ def testWrapperComputesTendency(self): def testWrapperComputesTendencyWithUnitConversion(self): state = { + 'time': timedelta(0), 'value': DataArray([0.011], attrs={'units': 'km'}) } tendencies, diagnostics = self.wrapped(state, timedelta(seconds=5)) diff --git a/tests/test_util.py b/tests/test_util.py index 26d7131..a2c692e 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -13,32 +13,27 @@ def same_list(list1, list2): [item in list2 for item in list1] + [item in list1 for item in list2])) -class MockPrognostic(Prognostic): +class PrognosticPropertiesContainer(object): - def __init__(self): - self._num_updates = 0 + def __init__(self, input_properties, tendency_properties, diagnostic_properties): + self.input_properties = input_properties + self.tendency_properties = tendency_properties + self.diagnostic_properties = diagnostic_properties - def __call__(self, state): - self._num_updates += 1 - return {}, {'num_updates': self._num_updates} +class ImplicitPropertiesContainer(object): -class MockImplicit(Implicit): + def __init__(self, input_properties, diagnostic_properties, output_properties): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self.output_properties = output_properties - def __init__(self): - self._a = 1 - def __call__(self, state): - return self._a +class DiagnosticPropertiesContainer(object): - -class MockDiagnostic(Diagnostic): - - def __init__(self): - self._a = 1 - - def __call__(self, state): - return self._a + def __init__(self, input_properties, diagnostic_properties): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties def test_update_dict_by_adding_another_adds_shared_arrays(): @@ -87,64 +82,342 @@ def test_get_component_aliases_with_no_args(): assert len(aliases.keys()) == 0 -def test_get_component_aliases_with_single_component_arg(): - components = [MockPrognostic(), MockImplicit(), MockDiagnostic()] - for c, comp in enumerate(components): - aliases = get_component_aliases(comp) - assert type(aliases) == dict - if c == 3: - assert len(aliases.keys()) == 2 - for k in ['T', 'P']: - assert k in list(aliases.values()) - else: - assert len(aliases.keys()) == 0 - - -class DummyProg1(Prognostic): - input_properties = {'temperature': {'alias': 'T'}} - tendency_properties = {'temperature': {'alias': 'TEMP'}} - - def __init__(self): - self._a = 1 - - def __call__(self, state): - return self._a - - -class DummyProg2(Prognostic): - input_properties = {'temperature': {'alias': 't'}} - - def __init__(self): - self._a = 1 - - def __call__(self, state): - return self._a - - -class DummyProg3(Prognostic): - input_properties = {'temperature': {}} - diagnostic_properties = {'pressure': {}} - tendency_properties = {'temperature': {}} - - def __init__(self): - self._a = 1 - - def __call__(self, state): - return self._a - - -def test_get_component_aliases_with_different_values(): - # two different aliases in the same Component: - aliases = get_component_aliases(DummyProg1()) - assert len(aliases.keys()) == 1 - assert aliases['temperature'] == 'TEMP' - # two different aliases in different Components: - aliases = get_component_aliases(DummyProg1(), DummyProg2()) - assert len(aliases.keys()) == 1 - assert aliases['temperature'] == 't' - # NO aliases in component - aliases = get_component_aliases(DummyProg3) - assert len(aliases.keys()) == 0 +def test_get_component_aliases_prognostic(): + aliases = get_component_aliases( + PrognosticPropertiesContainer( + input_properties={ + 'temperature': { + 'alias': 'T', + } + }, + tendency_properties={ + 'wind': { + 'alias': 'u' + } + }, + diagnostic_properties={ + 'specific_humidity': { + 'alias': 'q' + }, + } + ) + ) + assert aliases == {'temperature': 'T', 'wind': 'u', 'specific_humidity': 'q'} + + +def test_get_component_aliases_no_alias_prognostic(): + aliases = get_component_aliases( + PrognosticPropertiesContainer( + input_properties={ + 'temperature': { + } + }, + tendency_properties={ + 'wind': { + } + }, + diagnostic_properties={ + 'specific_humidity': { + }, + } + ) + ) + assert aliases == {} + + +def test_get_component_aliases_empty_prognostic(): + aliases = get_component_aliases( + PrognosticPropertiesContainer( + input_properties={}, + tendency_properties={}, + diagnostic_properties={} + ) + ) + assert aliases == {} + + +def test_get_component_aliases_diagnostic(): + aliases = get_component_aliases( + DiagnosticPropertiesContainer( + input_properties={ + 'temperature': { + 'alias': 'T', + } + }, + diagnostic_properties={ + 'specific_humidity': { + 'alias': 'q' + }, + } + ) + ) + assert aliases == {'temperature': 'T', 'specific_humidity': 'q'} + + +def test_get_component_aliases_no_alias_diagnostic(): + aliases = get_component_aliases( + DiagnosticPropertiesContainer( + input_properties={ + 'temperature': { + } + }, + diagnostic_properties={ + 'specific_humidity': { + }, + } + ) + ) + assert aliases == {} + + +def test_get_component_aliases_empty_diagnostic(): + aliases = get_component_aliases( + DiagnosticPropertiesContainer( + input_properties={}, + diagnostic_properties={} + ) + ) + assert aliases == {} + + +def test_get_component_aliases_implicit(): + aliases = get_component_aliases( + ImplicitPropertiesContainer( + input_properties={ + 'temperature': { + 'alias': 'T', + }, + 'input2': { + 'alias': 'in2', + } + }, + output_properties={ + 'wind': { + 'alias': 'u' + } + }, + diagnostic_properties={ + 'specific_humidity': { + 'alias': 'q' + }, + } + ) + ) + assert aliases == { + 'temperature': 'T', 'input2': 'in2', 'wind': 'u', 'specific_humidity': 'q'} + + +def test_get_component_aliases_no_alias_implicit(): + aliases = get_component_aliases( + ImplicitPropertiesContainer( + input_properties={ + 'temperature': {}, + }, + output_properties={ + 'wind': {} + }, + diagnostic_properties={ + 'specific_humidity': {}, + } + ) + ) + assert aliases == {} + + +def test_get_component_aliases_input_over_diagnostic(): + aliases = get_component_aliases( + ImplicitPropertiesContainer( + input_properties={ + 'temperature': {'alias': 'correct'}, + }, + output_properties={}, + diagnostic_properties={ + 'temperature': {'alias': 'incorrect'}, + } + ) + ) + assert aliases == {'temperature': 'correct'} + + +def test_get_component_aliases_input_over_output(): + aliases = get_component_aliases( + ImplicitPropertiesContainer( + input_properties={ + 'temperature': {'alias': 'correct'}, + }, + output_properties={ + 'temperature': {'alias': 'incorrect'}, + }, + diagnostic_properties={} + ) + ) + assert aliases == {'temperature': 'correct'} + + +def test_get_component_aliases_output_over_diagnostic(): + aliases = get_component_aliases( + ImplicitPropertiesContainer( + input_properties={ + }, + output_properties={ + 'temperature': {'alias': 'correct'}, + }, + diagnostic_properties={ + 'temperature': {'alias': 'not'}, + } + ) + ) + assert aliases == {'temperature': 'correct'} + + +def test_get_component_aliases_diagnostic_over_tendency(): + aliases = get_component_aliases( + PrognosticPropertiesContainer( + input_properties={ + }, + tendency_properties={ + 'temperature': {'alias': 'not'}, + }, + diagnostic_properties={ + 'temperature': {'alias': 'correct'}, + } + ) + ) + assert aliases == {'temperature': 'correct'} + + +def test_get_component_aliases_input_over_tendency(): + aliases = get_component_aliases( + PrognosticPropertiesContainer( + input_properties={ + 'temperature': {'alias': 'T'}, + }, + tendency_properties={ + 'temperature': {'alias': 'not'}, + }, + diagnostic_properties={ + } + ) + ) + assert aliases == {'temperature': 'T'} + + +def test_get_component_aliases_empty_implicit(): + aliases = get_component_aliases( + ImplicitPropertiesContainer( + input_properties={}, + output_properties={}, + diagnostic_properties={} + ) + ) + assert aliases == {} + + +def test_get_component_aliases_all_types_no_overlap(): + aliases = get_component_aliases( + ImplicitPropertiesContainer( + input_properties={ + 'input1': { + 'alias': 'in1', + } + }, + output_properties={ + 'output1': { + 'alias': 'out1' + } + }, + diagnostic_properties={ + 'diagnostic1': { + 'alias': 'diag1' + }, + } + ), + DiagnosticPropertiesContainer( + input_properties={ + 'input2': { + 'alias': 'in2', + } + }, + diagnostic_properties={ + 'diagnostic2': { + 'alias': 'diag2' + }, + } + ), + PrognosticPropertiesContainer( + input_properties={ + 'input3': { + 'alias': 'in3', + } + }, + tendency_properties={ + 'tendency3': { + 'alias': 'tend3' + } + }, + diagnostic_properties={ + 'diagnostic3': { + 'alias': 'diag3' + }, + } + ) + ) + assert aliases == { + 'input1': 'in1', 'output1': 'out1', 'diagnostic1': 'diag1', + 'input2': 'in2', 'diagnostic2': 'diag2', 'input3': 'in3', + 'tendency3': 'tend3', 'diagnostic3': 'diag3'} + +def test_get_component_aliases_all_types_with_overlap(): + aliases = get_component_aliases( + ImplicitPropertiesContainer( + input_properties={ + 'input1': { + 'alias': 'in1', + } + }, + output_properties={ + 'output1': { + 'alias': 'out1' + } + }, + diagnostic_properties={ + 'diagnostic1': { + 'alias': 'diag1' + }, + } + ), + DiagnosticPropertiesContainer( + input_properties={ + 'input1': { + 'alias': 'in1', + } + }, + diagnostic_properties={ + 'diagnostic1': { + 'alias': 'diag1' + }, + } + ), + PrognosticPropertiesContainer( + input_properties={ + 'input1': { + 'alias': 'in1', + } + }, + tendency_properties={ + 'tendency1': { + 'alias': 'tend1', + } + }, + diagnostic_properties={ + 'diagnostic1': { + 'alias': 'diag1' + }, + } + ) + ) + assert aliases == { + 'input1': 'in1', 'output1': 'out1', 'diagnostic1': 'diag1', 'tendency1': 'tend1'} def test_ensure_no_shared_keys_empty_dicts(): From 5568c4cf3dc7a0fa4e4a967abe8253dbac149e7c Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 23 Mar 2018 16:50:08 -0700 Subject: [PATCH 25/98] Removed blank line for flake8 --- sympl/_core/composite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 1eafcf1..fdaf43b 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -95,7 +95,6 @@ def __init__(self, *args): self.input_properties self.tendency_properties - def __call__(self, state): """ Gets tendencies and diagnostics from the passed model state. From ffdaba8fa1a867c6e0fa32bd06b97febdce83bc1 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 23 Mar 2018 16:52:16 -0700 Subject: [PATCH 26/98] Changed travis build priority --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cfa6b60..df1a762 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ # This file was autogenerated and will overwrite each time you run travis_pypi_setup.py env: -- TOXENV=cov -- TOXENV=flake8 - TOXENV=py35 - TOXENV=py27 +- TOXENV=flake8 +- TOXENV=cov install: - pip install -U tox language: python From 9606dd78d683cbeb199294b8d1210019f64586e5 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 26 Mar 2018 10:31:17 -0700 Subject: [PATCH 27/98] Extracted composite stuff to mixins --- sympl/_core/composite.py | 108 +++++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 44 deletions(-) diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 85c3386..daa635a 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -5,29 +5,56 @@ combine_component_properties) -class ComponentComposite(object): - """ - A composite of components that allows them to be called as one object. - - Attributes - ---------- - component_list: list - The components being composited by this object. - """ - - component_class = None +class InputPropertiesCompositeMixin(object): @property def input_properties(self): return combine_component_properties(self.component_list, 'input_properties') + def __init__(self, *args): + self.input_properties + super(InputPropertiesCompositeMixin, self).__init__() + + def _combine_attribute(self, attr): + return_attr = [] + for component in self.component_list: + return_attr.extend(getattr(component, attr)) + return tuple(set(return_attr)) # set to deduplicate + + +class DiagnosticPropertiesCompositeMixin(object): + @property def diagnostic_properties(self): return_dict = {} for component in self.component_list: + ensure_no_shared_keys(component.diagnostic_properties, return_dict) return_dict.update(component.diagnostic_properties) return return_dict + def __init__(self, *args): + self.diagnostic_properties + super(DiagnosticPropertiesCompositeMixin, self).__init__() + + def _combine_attribute(self, attr): + return_attr = [] + for component in self.component_list: + return_attr.extend(getattr(component, attr)) + return tuple(set(return_attr)) # set to deduplicate + + +class ComponentComposite(object): + """ + A composite of components that allows them to be called as one object. + + Attributes + ---------- + component_list: list + The components being composited by this object. + """ + + component_class = None + def __str__(self): return '{}(\n{}\n)'.format( self.__class__, @@ -49,27 +76,15 @@ def __init__(self, *args): ------ SharedKeyError If two components compute the same diagnostic quantity. + InvalidPropertyDictError + If two components require the same input or compute the same + output quantity, and their dimensions or units are incompatible + with one another. """ if self.component_class is not None: ensure_components_have_class(args, self.component_class) self.component_list = args - - def ensure_no_diagnostic_output_overlap(self): - diagnostic_names = [] - for component in self.component_list: - diagnostic_names.extend(component.diagnostic_properties.keys()) - for name in diagnostic_names: - if diagnostic_names.count(name) > 1: - raise SharedKeyError( - 'Two components in a composite should not compute ' - 'the same diagnostic, but multiple passed ' - 'components compute {}'.format(name)) - - def _combine_attribute(self, attr): - return_attr = [] - for component in self.component_list: - return_attr.extend(getattr(component, attr)) - return tuple(set(return_attr)) # set to deduplicate + super(ComponentComposite, self).__init__() def ensure_components_have_class(components, component_class): @@ -80,7 +95,9 @@ def ensure_components_have_class(components, component_class): component_class, type(component))) -class PrognosticComposite(ComponentComposite, Prognostic): +class PrognosticComposite( + ComponentComposite, InputPropertiesCompositeMixin, + DiagnosticPropertiesCompositeMixin, Prognostic): component_class = Prognostic @@ -89,11 +106,22 @@ def tendency_properties(self): return combine_component_properties(self.component_list, 'tendency_properties') def __init__(self, *args): - super(PrognosticComposite, self).__init__(*args) - self.ensure_tendency_outputs_are_compatible() - self.ensure_no_diagnostic_output_overlap() + """ + Args + ---- + *args + The components that should be wrapped by this object. - def ensure_tendency_outputs_are_compatible(self): + Raises + ------ + SharedKeyError + If two components compute the same diagnostic quantity. + InvalidPropertyDictError + If two components require the same input or compute the same + output quantity, and their dimensions or units are incompatible + with one another. + """ + super(PrognosticComposite, self).__init__(*args) self.tendency_properties def __call__(self, state): @@ -118,9 +146,6 @@ def __call__(self, state): Raises ------ - SharedKeyError - If multiple Prognostic objects contained in the - collection return the same diagnostic quantity. KeyError If a required quantity is missing from the state. InvalidStateError @@ -138,14 +163,12 @@ def array_call(self, state): raise NotImplementedError() -class DiagnosticComposite(ComponentComposite, Diagnostic): +class DiagnosticComposite( + ComponentComposite, InputPropertiesCompositeMixin, + DiagnosticPropertiesCompositeMixin, Diagnostic): component_class = Diagnostic - def __init__(self, *args): - super(DiagnosticComposite, self).__init__(*args) - self.ensure_no_diagnostic_output_overlap() - def __call__(self, state): """ Gets diagnostics from the passed model state. @@ -164,9 +187,6 @@ def __call__(self, state): Raises ------ - SharedKeyError - If multiple Diagnostic objects contained in the - collection return the same diagnostic quantity. KeyError If a required quantity is missing from the state. InvalidStateError From cfd57e28e37ab4743ac62ec9f4efe65c8c3d0666 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 26 Mar 2018 10:40:17 -0700 Subject: [PATCH 28/98] Removed unused import --- sympl/_core/composite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 46e70bd..05c54a6 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -1,4 +1,3 @@ -from .exceptions import SharedKeyError from .base_components import Prognostic, Diagnostic, Monitor from .util import ( update_dict_by_adding_another, ensure_no_shared_keys, From 2b312bd9f1aa7569603f90b910e03700b6968dfc Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 26 Mar 2018 11:12:33 -0700 Subject: [PATCH 29/98] Updated documentation for 0.4.0 --- docs/computation.rst | 3 + docs/constants.rst | 12 ++++ docs/overview.rst | 2 + docs/timestepping.rst | 6 +- docs/writing_components.rst | 134 +++++++++++++----------------------- 5 files changed, 69 insertions(+), 88 deletions(-) diff --git a/docs/computation.rst b/docs/computation.rst index cfa91c3..51822b1 100644 --- a/docs/computation.rst +++ b/docs/computation.rst @@ -24,6 +24,9 @@ In addition to the computational functionality below, all components have "prope for their inputs and outputs, which are described in the section :ref:`Input/Output Properties`. +Details on the internals of components and how to write them are in the section on +:ref:`Writing Components`. + Prognostic ---------- diff --git a/docs/constants.rst b/docs/constants.rst index 41390d6..a3f183b 100644 --- a/docs/constants.rst +++ b/docs/constants.rst @@ -25,6 +25,18 @@ imported by calling :py:func:`~sympl.reset_constants`. .. autofunction:: sympl.reset_constants +Debugging and Logging Constants +------------------------------- + +You can get a string describing current constants by calling :py:func:`~sympl.get_constants_string`: + +.. code-block:: python + + import sympl + print(sympl.get_constants_string()) + +.. autofunction:: sympl.get_constants_string + Condensible Quantities ---------------------- diff --git a/docs/overview.rst b/docs/overview.rst index af784a9..4f1b4b5 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -45,6 +45,8 @@ and packages to be used alongside one another. Then where's the model? ----------------------- +Check out `CliMT `_ as an example. + Models created with Sympl can work differently from traditional Fortran models. A model developer makes the components of their model available. Using these components, you can write a script which acts as the model executable, but also diff --git a/docs/timestepping.rst b/docs/timestepping.rst index 85ea6db..1f1a20d 100644 --- a/docs/timestepping.rst +++ b/docs/timestepping.rst @@ -3,12 +3,12 @@ Timestepping :py:class:`~sympl.TimeStepper` objects use time derivatives from :py:class:`~sympl.Prognostic` objects to step a model state forward in time. -They are initialized using a list of :py:class:`~sympl.Prognostic` objects. +They are initialized using any number of :py:class:`~sympl.Prognostic` objects. .. code-block:: python from sympl import AdamsBashforth - time_stepper = AdamsBashforth([MyPrognostic(), MyOtherPrognostic()]) + time_stepper = AdamsBashforth(MyPrognostic(), MyOtherPrognostic()) Once initialized, a :py:class:`~sympl.TimeStepper` object has a very similar interface to the :py:class:`~sympl.Implicit` object. @@ -16,7 +16,7 @@ interface to the :py:class:`~sympl.Implicit` object. .. code-block:: python from datetime import timedelta - time_stepper = AdamsBashforth([MyPrognostic()]) + time_stepper = AdamsBashforth(MyPrognostic()) timestep = timedelta(minutes=10) diagnostics, next_state = time_stepper(state, timestep) state.update(diagnostics) diff --git a/docs/writing_components.rst b/docs/writing_components.rst index 6056fbe..7de3de9 100644 --- a/docs/writing_components.rst +++ b/docs/writing_components.rst @@ -14,7 +14,8 @@ Writing an Example ------------------ Let's start with a Prognostic component which relaxes temperature towards some -target temperature. +target temperature. We'll go over the sections of this example step-by-step +below. .. code-block:: python @@ -45,24 +46,15 @@ target temperature. } } - def __init__(self, tau=1., target_temperature=300.): - self._tau = tau - self._T0 = target_temperature - - def __call__(self, state): - # we get numpy arrays with specifications from input_properties - raw_arrays = get_numpy_arrays_with_properties( - state, self.input_properties) - T = raw_arrays['air_temperature'] - # here the actual computation happens - raw_tendencies = { - 'air_temperature': (T - self._T0)/self._tau, + def __init__(self, damping_timescale_seconds=1., target_temperature_K=300.): + self._tau = damping_timescale_seconds + self._T0 = target_temperature_K + + def array_call(self, state): + tendencies = { + 'air_temperature': (state['air_temperature'] - self._T0)/self._tau, } - # now we re-format the data in a way the host model can use diagnostics = {} - tendencies = restore_data_arrays_with_properties( - raw_tendencies, self.tendency_properties, - state, self.input_properties) return tendencies, diagnostics Imports @@ -142,13 +134,11 @@ or on an instance, as when you do: These properties are described in :ref:`Component Types`. They are very useful! They clearly document your code. Here we can see that air_temperature will be used as a 1-dimensional flattened array in units of degrees Kelvin. Sympl -can also understand these properties, and use them to automatically -acquire arrays in the dimensions and units that you need. -It can also test thatsome of these properties are accurate. -It's your responsibility, though, to make sure that the input units are the -units you want to acquire in the numpy array data, and that the output units -are the units of the values in the raw output arrays that you want to convert -to :py:class:`~sympl.DataArray` objects. +uses these properties to automatically acquire arrays in the dimensions and +units that you need, and to automatically convert your output back into a +form consistent with the dimensions of the model state. It will warn you if +you create extra outputs which are not defined in the properties, or if there +is an output defined in the properties that is missing. It is possible that some of these attributes won't be known until you create the object (they may depend on things passed in on initialization). @@ -164,14 +154,9 @@ weird name: .. code-block:: python - def __init__(self, damping_timescale=1., target_temperature=300.): - """ - damping_timescale is the damping timescale in seconds. - target_temperature is the temperature that will be relaxed to, - in degrees Kelvin. - """ - self._tau = damping_timescale - self._T0 = target_temperature + def __init__(self, damping_timescale_seconds=1., target_temperature_K=300.): + self._tau = damping_timescale_seconds + self._T0 = target_temperature_K This is the function that is called when you create an instance of your object. All methods on objects take in a first argument called ``self``. You don't see @@ -192,8 +177,8 @@ is also optimal. What these attributes mean is clearly defined in the two lines: .. code-block:: python - self._tau = damping_timescale - self._T0 = target_temperature + self._tau = damping_timescale_seconds + self._T0 = target_temperature_K Obviously ``self._tau`` is the damping timescale, and ``self._T0`` is the target temperature for the relaxation. Now you can use these shorter variables @@ -203,57 +188,45 @@ variables are well-documented. The Computation *************** -That brings us to the ``__call__`` method. This is what's called when you -use the object as though it is a function. In Sympl components, this is the -method which takes in a state dictionary and returns dictionaries with outputs. +That brings us to the ``array_call`` method. In Sympl components, this is the +method which takes in a state dictionary as numpy arrays (*not* ``DataArray``s) +and returns dictionaries with numpy array outputs. .. code-block:: python - def __call__(self, state): - # we get numpy arrays with specifications from input_properties - raw_arrays = get_numpy_arrays_with_properties( - state, self.input_properties) - T = raw_arrays['air_temperature'] - # here the actual computation happens - raw_tendencies = { - 'air_temperature': (T - self._T0)/self._tau, + def array_call(self, state): + tendencies = { + 'air_temperature': (state['air_temperature'] - self._T0)/self._tau, } - # now we re-format the data in a way the host model can use diagnostics = {} - tendencies = restore_data_arrays_with_properties( - raw_tendencies, self.tendency_properties, - state, self.input_properties) - return diagnostics, tendencies - -There are two helper functions used in this code that we strongly recommend -using. They take care of the work of making sure you get variables that are -in the units your component needs, and have the dimensions your component needs. - -:py:func:`~sympl.get_numpy_arrays_with_properties` uses the input_properties -dictionary you give it to extract numpy arrays with those properties from the -input state. It will convert units to ensure the numbers are in the specified + return tendencies, diagnostics + +Sympl will automatically handle taking in the input state of ``DataArray`` +objects and converting it to the form defined by the ``input_properties`` of your +component. It will convert units to ensure the numbers are in the specified units, and it will reshape the data to give it the shape specified in ``dims``. -For example, if dims is ``['*', 'z']`` then it will give you a 2-dimensional array -whose second axis is the vertical, and first axis is a flattening of any other -dimensions. If you specify ``['*', 'mid_levels']`` then the result is similar, but -only 'mid_levels' is an acceptable vertical dimension. The ``match_dims_like`` +For example, if dims is ``['*', 'mid_levels']`` then it will give you a +2-dimensional array whose second axis is the vertical on mid levels, and first +axis is a flattening of any other dimensions. The ``match_dims_like`` property on ``air_pressure`` tells Sympl that any wildcard-matched dimensions -(ones that match 'x', 'y', 'z', or '*') should be the same between the two +('*') should be the same between the two quantities, meaning they're on the same grid for those wildcards. You can still, however, have one be on say 'mid_levels' and another on 'interface_levels' if -those dimensions are explicitly listed (instead of listing 'z'). - -:py:func:`~sympl.restore_data_arrays_with_properties` does something fairly -magical. In this example, it takes the raw_tendencies dictionary and converts -the value for 'air_temperature' from a numpy array to a DataArray that has -the same dimensions as ``air_temperature`` had in the input state. That means -that you could pass this object a state with whatever dimensions you want, -whether it's (x, y, z), or (z, x, y), or (x, y), or (station_number, z), etc. +those dimensions are explicitly listed. + +After you return dictionaries of numpy arrays, Sympl will convert these outputs +back to ``DataArray`` objects. In this example, it takes the tendencies +dictionary and converts the value for 'air_temperature' from a numpy array to a +``DataArray`` that has the same dimensions as ``air_temperature`` had in the +input state. That means that you could pass this object a state with whatever +dimensions you want, whether it's ``('longitude', 'latitude', 'mid_levels')``, or +``('interface_levels',)`` or ``('station_number', 'planet_number')``, etc. and this component will be able to take in that state, and return a tendency dictionary with the same dimensions (and order) that the model uses! And internally you can work with a simple 1-dimensional array. This is particularly useful for writing pointwise components using ``['*']`` or column -components with ``['*', 'z']`` or ``['z', '*']``. +components with, for example, ``['*', 'mid_levels']`` or +``['interface_levels', '*']``. You can read more about properties in the section :ref:`Input/Output Properties`. @@ -294,21 +267,12 @@ computational code, we can write: .. code-block:: python - def __call__(self, state): - # we get numpy arrays with specifications from input_properties - raw_arrays = get_numpy_arrays_with_properties( - state, self.input_properties) - T = raw_arrays['T'] - # here the actual computation happens - raw_tendencies = { - 'T': (T - self._T0)/self._tau, + def array_call(self, state): + tendencies = { + 'T': (state['T'] - self._T0)/self._tau, } - # now we re-format the data in a way the host model can use diagnostics = {} - tendencies = restore_data_arrays_with_properties( - raw_tendencies, self.tendency_properties, - state, self.input_properties) - return diagnostics, tendencies + return tendencies, diagnostics Instead of using 'air_temperature' in the raw_arrays and raw_tendencies dictionaries, we can use 'T'. This doesn't matter much for a name as short as From 0384fbff7606c10278629aed7ea183fe98bab1b2 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Tue, 27 Mar 2018 15:23:02 -0700 Subject: [PATCH 30/98] Updated * wildcard behavior, more tests and bug fixes Will now guarantee that * matches the same dimensions across quantities. dims_like is no longer used. --- HISTORY.rst | 7 + setup.py | 2 +- sympl/__init__.py | 5 +- sympl/_core/base_components.py | 3 +- sympl/_core/composite.py | 9 +- sympl/_core/state.py | 191 +++++++++++++++ sympl/_core/timestepper.py | 3 +- sympl/_core/util.py | 27 +- tests/test_composite.py | 340 ++++++++++++++++++++++++++ tests/test_get_restore_numpy_array.py | 143 ++--------- tests/test_timestepping.py | 3 +- 11 files changed, 590 insertions(+), 143 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2f5bf53..c7020f6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -59,6 +59,13 @@ Breaking changes proper conversion of dimensions and units. * Removed ComponentTestBase from package. All of its tests except for output caching are now performed on object initialization or call time. +* "*" matches are now enforced to be the same across all quantities of a + component, such that the length of the "*" axis will be the same for all + quantities. Any missing dimensions that are present on other quantities + will be created and broadcast to achieve this. +* dims_like is obsolete as a result, and is no longer used. `dims` should be + used instead. If present, `dims` from input properties will be used as + default. v0.3.2 ------ diff --git a/setup.py b/setup.py index 66154a5..95b58fc 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ requirements = [ 'numpy>=1.10', 'pint>=0.7.0', - 'xarray>=0.8.0', + 'xarray>=0.9.3', 'six', ] diff --git a/sympl/__init__.py b/sympl/__init__.py index bb69dc1..c79e92d 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -17,9 +17,10 @@ from ._core.util import ( ensure_no_shared_keys, get_numpy_array, jit, - restore_dimensions, get_numpy_arrays_with_properties, - restore_data_arrays_with_properties, + restore_dimensions, get_component_aliases) +from ._core.state import (get_numpy_arrays_with_properties, + restore_data_arrays_with_properties) from ._components import ( PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index d9afa0a..f19a039 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -1,6 +1,5 @@ import abc -from .util import ( - get_numpy_arrays_with_properties, restore_data_arrays_with_properties) +from .state import get_numpy_arrays_with_properties, restore_data_arrays_with_properties from .time import timedelta from .exceptions import ( InvalidPropertyDictError, ComponentExtraOutputError, diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 05c54a6..925baf0 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -2,6 +2,7 @@ from .util import ( update_dict_by_adding_another, ensure_no_shared_keys, combine_component_properties) +from .exceptions import InvalidPropertyDictError class InputPropertiesCompositeMixin(object): @@ -27,9 +28,12 @@ class DiagnosticPropertiesCompositeMixin(object): def diagnostic_properties(self): return_dict = {} for component in self.component_list: - print(component.diagnostic_properties.keys(), return_dict.keys()) ensure_no_shared_keys(component.diagnostic_properties, return_dict) return_dict.update(component.diagnostic_properties) + for name, properties in return_dict.items(): + if 'dims' not in properties.keys() and not (name in self.input_properties): + raise InvalidPropertyDictError( + 'Must define dims for diagnostic output {}'.format(name)) return return_dict def __init__(self, *args): @@ -103,7 +107,8 @@ class PrognosticComposite( @property def tendency_properties(self): - return combine_component_properties(self.component_list, 'tendency_properties') + return combine_component_properties( + self.component_list, 'tendency_properties', self.input_properties) def __init__(self, *args): """ diff --git a/sympl/_core/state.py b/sympl/_core/state.py index 45fc154..05a5ec4 100644 --- a/sympl/_core/state.py +++ b/sympl/_core/state.py @@ -1,3 +1,7 @@ +from .exceptions import InvalidStateError, InvalidPropertyDictError +import numpy as np +from .array import DataArray + def copy_untouched_quantities(old_state, new_state): for key in old_state.keys(): @@ -27,3 +31,190 @@ def multiply(scalar, state): if hasattr(out_state[key], 'attrs'): out_state[key].attrs = state[key].attrs return out_state + + +def get_wildcard_matches_and_dim_lengths(state, property_dictionary): + wildcard_names = [] + dim_lengths = {} + # Loop to get the set of names matching "*" (wildcard names) + for quantity_name, properties in property_dictionary.items(): + ensure_properties_have_dims_and_units(properties, quantity_name) + for dim_name, length in zip(state[quantity_name].dims, state[quantity_name].shape): + if dim_name not in dim_lengths.keys(): + dim_lengths[dim_name] = length + elif length != dim_lengths[dim_name]: + raise InvalidStateError( + 'Dimension {} conflicting lengths {} and {} in different ' + 'state quantities.'.format(dim_name, length, dim_lengths[dim_name])) + new_wildcard_names = [ + dim for dim in state[quantity_name].dims if dim not in properties['dims']] + if len(new_wildcard_names) > 0 and '*' not in properties['dims']: + raise InvalidStateError( + 'Quantity {} has unexpected dimensions {}.'.format( + quantity_name, new_wildcard_names)) + wildcard_names.extend( + [name for name in new_wildcard_names if name not in wildcard_names]) + wildcard_names = tuple(wildcard_names) + return wildcard_names, dim_lengths + + +def get_numpy_arrays_with_properties(state, property_dictionary): + out_dict = {} + wildcard_names, dim_lengths = get_wildcard_matches_and_dim_lengths( + state, property_dictionary) + # Now we actually retrieve output arrays since we know the precise out dims + for name, properties in property_dictionary.items(): + ensure_quantity_has_units(state[name], name) + try: + quantity = state[name].to_units(properties['units']) + except ValueError: + raise InvalidStateError( + 'Could not convert quantity {} from units {} to units {}'.format( + name, state[name].attrs['units'], properties['units'] + ) + ) + out_dims = [] + out_dims.extend(properties['dims']) + has_wildcard = '*' in out_dims + if has_wildcard: + i_wildcard = out_dims.index('*') + out_dims[i_wildcard:i_wildcard+1] = wildcard_names + out_array = get_numpy_array( + quantity, out_dims=out_dims, dim_lengths=dim_lengths) + if has_wildcard: + out_array = flatten_wildcard_dims( + out_array, i_wildcard, i_wildcard + len(wildcard_names)) + if 'alias' in properties.keys(): + out_name = properties['alias'] + else: + out_name = name + out_dict[out_name] = out_array + return out_dict + + +def flatten_wildcard_dims(array, i_start, i_end): + if i_end > len(array.shape): + raise ValueError('i_end should be less than the number of axes in array') + elif i_start < 0: + raise ValueError('i_start should be greater than 0') + elif i_start > i_end: + raise ValueError('i_start should be less than or equal to i_end') + elif i_start == i_end: + # We need to insert a singleton dimension at i_start + target_shape = [] + target_shape.extend(array.shape) + target_shape.insert(i_start, 1) + else: + target_shape = [] + wildcard_length = 1 + for i, length in enumerate(array.shape): + if i_start <= i < i_end: + wildcard_length *= length + else: + target_shape.append(length) + if i == i_end - 1: + target_shape.append(wildcard_length) + return array.reshape(target_shape) + + +def get_numpy_array(data_array, out_dims, dim_lengths): + """ + Gets a numpy array from the data_array with the desired out_dims, and a + dict of dim_lengths that will give the length of any missing dims in the + data_array. + """ + if len(data_array.values.shape) == 0 and len(out_dims) == 0: + return data_array.values # special case, 0-dimensional scalar array + else: + missing_dims = [dim for dim in out_dims if dim not in data_array.dims] + for dim in missing_dims: + data_array = data_array.expand_dims(dim) + numpy_array = data_array.transpose(*out_dims).values + if len(missing_dims) == 0: + out_array = numpy_array + else: # expand out missing dims which are currently length 1. + out_shape = [dim_lengths.get(name, 1) for name in out_dims] + if out_shape == list(numpy_array.shape): + out_array = numpy_array + else: + out_array = np.empty(out_shape, dtype=numpy_array.dtype) + out_array[:] = numpy_array + return out_array + + +def restore_data_arrays_with_properties( + raw_arrays, output_properties, input_state, input_properties): + wildcard_names, dim_lengths = get_wildcard_matches_and_dim_lengths( + input_state, input_properties) + for name, value in raw_arrays.items(): + if not isinstance(value, np.ndarray): + raw_arrays[name] = np.asarray(value) + out_dims_property = {} + for name, properties in output_properties.items(): + if 'dims' in properties.keys(): + out_dims_property[name] = properties['dims'] + elif name not in input_properties.keys(): + raise InvalidPropertyDictError( + 'Output dims must be specified for {} in properties'.format(name)) + elif 'dims' not in input_properties[name].keys(): + raise InvalidPropertyDictError( + 'Input dims must be specified for {} in properties'.format(name)) + else: + out_dims_property[name] = input_properties[name]['dims'] + out_dict = {} + for name, dims in out_dims_property.items(): + if 'alias' in output_properties[name].keys(): + raw_name = output_properties[name]['alias'] + elif name in input_properties.keys() and 'alias' in input_properties[name].keys(): + raw_name = input_properties[name]['alias'] + else: + raw_name = name + if '*' in dims: + i_wildcard = dims.index('*') + target_shape = [] + out_dims = [] + for i, length in enumerate(raw_arrays[raw_name].shape): + if i == i_wildcard: + target_shape.extend([dim_lengths[n] for n in wildcard_names]) + out_dims.extend(wildcard_names) + else: + target_shape.append(length) + out_dims.append(dims[i]) + out_array = np.reshape(raw_arrays[raw_name], target_shape) + else: + print(raw_arrays) + if len(dims) != len(raw_arrays[raw_name].shape): + raise InvalidPropertyDictError( + 'Returned array for {} has shape {} ' + 'which is incompatible with dims {} in properties'.format( + name, raw_arrays[raw_name].shape, dims)) + for dim, length in zip(dims, raw_arrays[raw_name].shape): + if dim in dim_lengths.keys() and dim_lengths[dim] != length: + raise InvalidPropertyDictError( + 'Dimension {} of quantity {} has length {}, but ' + 'another quantity has length {}'.format( + dim, name, length, dim_lengths[dim]) + ) + out_dims = dims + out_array = raw_arrays[raw_name] + out_dict[name] = DataArray( + out_array, + dims=out_dims, + attrs={'units': output_properties[name]['units']} + ) + return out_dict + + +def ensure_properties_have_dims_and_units(properties, quantity_name): + if 'dims' not in properties: + raise InvalidPropertyDictError( + 'dims not specified for quantity {}'.format(quantity_name)) + if 'units' not in properties: + raise InvalidPropertyDictError( + 'units not specified for quantity {}'.format(quantity_name)) + + +def ensure_quantity_has_units(quantity, quantity_name): + if 'units' not in quantity.attrs: + raise InvalidStateError( + 'quantity {} is missing units attribute'.format(quantity_name)) diff --git a/sympl/_core/timestepper.py b/sympl/_core/timestepper.py index 83ff010..55448ea 100644 --- a/sympl/_core/timestepper.py +++ b/sympl/_core/timestepper.py @@ -53,7 +53,7 @@ class TimeStepper(object): def input_properties(self): input_properties = combine_component_properties( self.prognostic_list, 'input_properties') - return combine_properties(input_properties, self.output_properties) + return combine_properties([input_properties, self.output_properties]) @property def diagnostic_properties(self): @@ -188,6 +188,7 @@ def _insert_tendencies_to_diagnostics( 'TimeStepper ({}). You must disable ' 'tendencies_in_diagnostics for this TimeStepper.'.format( tendency_name)) + print(name, input_properties, output_properties) base_units = input_properties[name]['units'] diagnostics[tendency_name] = ( (new_state[name].to_units(base_units) - state[name].to_units(base_units)) / diff --git a/sympl/_core/util.py b/sympl/_core/util.py index 42b6aad..e149631 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -637,20 +637,27 @@ def get_final_shape(data_array, out_dims, direction_to_names): return final_shape -def combine_component_properties(component_list, property_name): - args = [] +def combine_component_properties(component_list, property_name, input_properties=None): + property_list = [] for component in component_list: - args.append(getattr(component, property_name)) - return combine_properties(*args) + property_list.append(getattr(component, property_name)) + return combine_properties(property_list, input_properties) -def combine_properties(*args): +def combine_properties(property_list, input_properties=None): + if input_properties is None: + input_properties = {} return_dict = {} - for property_dict in args: + for property_dict in property_list: for name, properties in property_dict.items(): if name not in return_dict: return_dict[name] = {} return_dict[name].update(properties) + if 'dims' not in properties.keys(): + if name in input_properties.keys() and 'dims' in input_properties[name].keys(): + return_dict[name]['dims'] = input_properties[name]['dims'] + else: + raise InvalidPropertyDictError() elif not units_are_compatible( properties['units'], return_dict[name]['units']): raise InvalidPropertyDictError( @@ -659,8 +666,14 @@ def combine_properties(*args): return_dict[name]['units'], properties['units'], name)) else: + if 'dims' in properties.keys(): + new_dims = properties['dims'] + elif name in input_properties.keys() and 'dims' in input_properties[name].keys(): + new_dims = input_properties[name]['dims'] + else: + raise InvalidPropertyDictError() try: - dims = combine_dims(return_dict[name]['dims'], properties['dims']) + dims = combine_dims(return_dict[name]['dims'], new_dims) return_dict[name]['dims'] = dims except InvalidPropertyDictError as err: raise InvalidPropertyDictError( diff --git a/tests/test_composite.py b/tests/test_composite.py index 2e411d4..6d29218 100644 --- a/tests/test_composite.py +++ b/tests/test_composite.py @@ -1,4 +1,5 @@ import pytest +import unittest import mock from sympl import ( Prognostic, Diagnostic, Monitor, PrognosticComposite, DiagnosticComposite, @@ -307,6 +308,44 @@ def test_diagnostic_composite_single_full_component(): assert composite.diagnostic_properties == diagnostic_properties +def test_diagnostic_composite_single_component_no_dims_on_diagnostic(): + input_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + }, + } + diagnostic_properties = { + 'diag1': { + 'units': 'km', + }, + } + diagnostic_output = {} + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, diagnostic_output) + composite = DiagnosticComposite(diagnostic) + assert composite.input_properties == input_properties + assert composite.diagnostic_properties == diagnostic_properties + + +def test_diagnostic_composite_single_component_missing_dims_on_diagnostic(): + input_properties = {} + diagnostic_properties = { + 'diag1': { + 'units': 'km', + }, + } + diagnostic_output = {} + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, diagnostic_output) + try: + DiagnosticComposite(diagnostic) + except InvalidPropertyDictError: + pass + else: + raise AssertionError('Should have raised InvalidPropertyDictError') + + def test_diagnostic_composite_two_components_no_overlap(): diagnostic1 = MockDiagnostic( input_properties={ @@ -571,6 +610,307 @@ def test_prognostic_composite_single_tendency(): assert composite.tendency_properties == prognostic.tendency_properties +def test_prognostic_composite_implicit_dims(): + prognostic = MockPrognostic( + input_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic) + assert composite.input_properties == prognostic.input_properties + assert composite.diagnostic_properties == prognostic.diagnostic_properties + assert composite.tendency_properties == { + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + } + + +def test_two_prognostic_composite_implicit_dims(): + prognostic = MockPrognostic( + input_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic, prognostic2) + assert composite.input_properties == prognostic.input_properties + assert composite.diagnostic_properties == prognostic.diagnostic_properties + assert composite.tendency_properties == { + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + } + + +def test_prognostic_composite_explicit_dims(): + prognostic = MockPrognostic( + input_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic) + assert composite.input_properties == prognostic.input_properties + assert composite.diagnostic_properties == prognostic.diagnostic_properties + assert composite.tendency_properties == prognostic.tendency_properties + + +def test_two_prognostic_composite_explicit_dims(): + prognostic = MockPrognostic( + input_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic, prognostic2) + assert composite.input_properties == prognostic.input_properties + assert composite.diagnostic_properties == prognostic.diagnostic_properties + assert composite.tendency_properties == prognostic.tendency_properties + + +def test_two_prognostic_composite_explicit_and_implicit_dims(): + prognostic = MockPrognostic( + input_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic, prognostic2) + assert composite.input_properties == prognostic.input_properties + assert composite.diagnostic_properties == prognostic.diagnostic_properties + assert composite.tendency_properties == prognostic.tendency_properties + + +def test_prognostic_composite_explicit_dims_not_in_input(): + prognostic = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic) + assert composite.input_properties == prognostic.input_properties + assert composite.diagnostic_properties == prognostic.diagnostic_properties + assert composite.tendency_properties == prognostic.tendency_properties + + +def test_two_prognostic_composite_incompatible_dims(): + prognostic = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + }, + 'input2': { + 'dims': ['dims3', 'dims1'], + 'units': 'degK / day' + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + }, + 'input2': { + 'dims': ['dims3', 'dims1'], + 'units': 'degK / day' + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dims3', 'dims1'], + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + try: + PrognosticComposite(prognostic, prognostic2) + except InvalidPropertyDictError: + pass + else: + raise AssertionError('Should have raised InvalidPropertyDictError') + + +def test_two_prognostic_composite_compatible_dims(): + prognostic = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + }, + 'input2': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day' + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + prognostic2 = MockPrognostic( + input_properties={ + 'input1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + }, + 'input2': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day' + } + }, + diagnostic_properties={}, + tendency_properties={ + 'tend1': { + 'dims': ['dims1', 'dims2'], + 'units': 'degK / day', + } + }, + diagnostic_output={}, + tendency_output={}, + ) + composite = PrognosticComposite(prognostic, prognostic2) + assert composite.input_properties == prognostic.input_properties + assert composite.diagnostic_properties == prognostic.diagnostic_properties + assert composite.tendency_properties == prognostic.tendency_properties + + def test_prognostic_composite_two_components_input(): prognostic1 = MockPrognostic( input_properties={ diff --git a/tests/test_get_restore_numpy_array.py b/tests/test_get_restore_numpy_array.py index fdb9c75..f46a839 100644 --- a/tests/test_get_restore_numpy_array.py +++ b/tests/test_get_restore_numpy_array.py @@ -525,12 +525,12 @@ def test_returns_numpy_array_alias_doesnt_apply_to_state(self): ), } try: - return_value = get_numpy_arrays_with_properties( + get_numpy_arrays_with_properties( state, property_dictionary) - except InvalidStateError: + except KeyError: pass else: - raise AssertionError('should have raised InvalidStateError') + raise AssertionError('should have raised KeyError') def test_returns_scalar_array(self): T_array = np.array(0.) @@ -662,35 +662,6 @@ def test_all_requested_properties_are_returned(self): assert np.all(return_value['air_temperature'] == 0.) assert np.all(return_value['air_pressure'] == 0.) - def test_raises_exception_on_missing_quantity(self): - property_dictionary = { - 'air_temperature': { - 'dims': ['x', 'y', 'z'], - 'units': 'degK', - }, - 'air_pressure': { - 'dims': ['x', 'y', 'z'], - 'units': 'Pa', - }, - } - state = { - 'air_temperature': DataArray( - np.zeros([4], dtype=np.float64), - dims=['z'], - attrs={'units': 'degK'}, - ), - 'eastward_wind': DataArray( - np.zeros([2,2,4], dtype=np.float64), - attrs={'units': 'm/s'} - ), - } - try: - get_numpy_arrays_with_properties(state, property_dictionary) - except InvalidStateError: - pass - else: - raise AssertionError('should have raised InvalidStateError') - def test_converts_units(self): property_dictionary = { 'air_temperature': { @@ -847,10 +818,6 @@ def test_collects_horizontal_dimensions(self): } } return_value = get_numpy_arrays_with_properties(input_state, input_properties) - assert np.byte_bounds( - return_value['air_temperature']) == np.byte_bounds( - T_array) - assert return_value['air_temperature'].base is T_array assert return_value['air_temperature'].shape == (4, 6) for i in range(3): for j in range(2): @@ -940,7 +907,7 @@ def test_returns_simple_value(self): raw_arrays = {key + '_tendency': value for key, value in raw_arrays.items()} output_properties = { 'air_temperature_tendency': { - 'dims_like': 'air_temperature', + 'dims': ['x', 'y', 'z'], 'units': 'degK/s', } } @@ -1013,7 +980,7 @@ def test_restores_collected_horizontal_dimensions(self): raw_arrays = {key + '_tendency': value for key, value in raw_arrays.items()} output_properties = { 'air_temperature_tendency': { - 'dims_like': 'air_temperature', + 'dims': ['z', '*'], 'units': 'degK/s', } } @@ -1029,52 +996,10 @@ def test_restores_collected_horizontal_dimensions(self): input_state['air_temperature'].values) assert (return_value['air_temperature_tendency'].values.base is input_state['air_temperature'].values) - assert return_value['air_temperature_tendency'].shape == (3, 2, 4) - assert np.all(return_value['air_temperature_tendency'] == T_array) - assert return_value['air_temperature_tendency'].dims == input_state['air_temperature'].dims - - def test_restores_coords(self): - x = np.array([0., 10.]) - y = np.array([0., 10.]) - z = np.array([0., 5., 10., 15.]) - input_state = { - 'air_temperature': DataArray( - np.zeros([2, 2, 4]), - dims=['x', 'y', 'z'], - attrs={'units': 'degK'}, - coords=[ - ('x', x, {'units': 'm'}), - ('y', y, {'units': 'km'}), - ('z', z, {'units': 'cm'})] - ) - } - input_properties = { - 'air_temperature': { - 'dims': ['x', 'y', 'z'], - 'units': 'degK', - } - } - raw_arrays = get_numpy_arrays_with_properties(input_state, input_properties) - raw_arrays = {key + '_tendency': value for key, value in raw_arrays.items()} - output_properties = { - 'air_temperature_tendency': { - 'dims_like': 'air_temperature', - 'units': 'degK/s', - } - } - return_value = restore_data_arrays_with_properties( - raw_arrays, output_properties, input_state, input_properties - ) - assert np.all(return_value['air_temperature_tendency'].coords['x'] == - input_state['air_temperature'].coords['x']) - assert return_value['air_temperature_tendency'].coords['x'].attrs['units'] == 'm' - assert np.all(return_value['air_temperature_tendency'].coords['y'] == - input_state['air_temperature'].coords['y']) - assert return_value['air_temperature_tendency'].coords['y'].attrs['units'] == 'km' - assert np.all(return_value['air_temperature_tendency'].coords['z'] == - input_state['air_temperature'].coords['z']) - assert return_value['air_temperature_tendency'].coords['z'].attrs['units'] == 'cm' - assert return_value['air_temperature_tendency'].dims == input_state['air_temperature'].dims + assert return_value['air_temperature_tendency'].dims == ('z', 'x', 'y') + assert return_value['air_temperature_tendency'].shape == (4, 3, 2) + for i in range(4): + assert np.all(return_value['air_temperature_tendency'][i, :, :] == T_array[:, :, i]) def test_restores_scalar_array(self): T_array = np.array(0.) @@ -1124,7 +1049,7 @@ def test_raises_on_invalid_shape(self): } output_properties = { 'foo': { - 'dims_like': 'air_temperature', + 'dims': ['x', 'y', 'z'], 'units': 'm', } } @@ -1156,22 +1081,22 @@ def test_raises_on_raw_array_missing(self): } output_properties = { 'foo': { - 'dims_like': 'air_temperature', + 'dims': ['x', 'y', 'z'], 'units': 'm', }, 'bar': { - 'dims_like': 'air_temperature', + 'dims': ['x', 'y', 'z'], 'units': 'm', - } + }, } try: restore_data_arrays_with_properties( raw_arrays, output_properties, input_state, input_properties ) - except ValueError: + except KeyError: pass else: - raise AssertionError('should have raised ValueError') + raise AssertionError('should have raised KeyError') def test_restores_aliased_name(self): input_state = { @@ -1192,7 +1117,7 @@ def test_restores_aliased_name(self): } output_properties = { 'air_pressure': { - 'dims_like': 'air_temperature', + 'dims': ['x', 'y', 'z'], 'units': 'm', 'alias': 'p', }, @@ -1205,40 +1130,6 @@ def test_restores_aliased_name(self): assert np.all(data_arrays['air_pressure'].values == raw_arrays['p']) assert np.byte_bounds(data_arrays['air_pressure'].values) == np.byte_bounds(raw_arrays['p']) - def test_restores_when_name_has_alias(self): - input_state = { - 'air_temperature': DataArray( - np.zeros([2, 2, 4]), - dims=['x', 'y', 'z'], - attrs={'units': 'degK'}, - ) - } - input_properties = { - 'air_temperature': { - 'dims': ['x', 'y', 'z'], - 'units': 'degK', - } - } - raw_arrays = { - 'air_pressure': np.zeros([2, 2, 4]) - } - output_properties = { - 'air_pressure': { - 'dims_like': 'air_temperature', - 'units': 'm', - 'alias': 'p', - }, - } - data_arrays = restore_data_arrays_with_properties( - raw_arrays, output_properties, input_state, input_properties - ) - assert len(data_arrays.keys()) == 1 - assert 'air_pressure' in data_arrays.keys() - assert np.all(data_arrays['air_pressure'].values == raw_arrays['air_pressure']) - assert np.byte_bounds( - data_arrays['air_pressure'].values) == np.byte_bounds( - raw_arrays['air_pressure']) - def test_restores_using_alias_from_input(self): input_state = { 'air_temperature': DataArray( @@ -1268,7 +1159,7 @@ def test_restores_using_alias_from_input(self): } output_properties = { 'air_pressure': { - 'dims_like': 'air_temperature', + 'dims': ['x', 'y', 'z'], 'units': 'm', }, } diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index 73ea226..def5114 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -60,7 +60,7 @@ def test_unused_quantities_carried_over(self): def test_timestepper_reveals_prognostics(self): prog1 = MockEmptyPrognostic() - prog1.input_properties = {'input1': {}} + prog1.input_properties = {'input1': {'dims': ['dim1'], 'units': 'm'}} time_stepper = self.timestepper_class(prog1) assert same_list(time_stepper.prognostic_list, (prog1,)) @@ -455,7 +455,6 @@ def test_copies_untouched_quantities(self): } _, new_state = stepper(state, timedelta(seconds=5)) assert 'input1' in new_state.keys() - print(new_state['input1'].values.data, untouched_quantity.values.data) assert new_state['input1'].dims == untouched_quantity.dims assert np.allclose(new_state['input1'].values, 10.) assert new_state['input1'].attrs['units'] == 'J' From 3de867cdbd1d2cfa669a58cd4eadfa4c5c401533 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Tue, 27 Mar 2018 15:24:57 -0700 Subject: [PATCH 31/98] Fixed indentation for flake8 --- sympl/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sympl/__init__.py b/sympl/__init__.py index c79e92d..500c1f8 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -19,7 +19,8 @@ get_numpy_array, jit, restore_dimensions, get_component_aliases) -from ._core.state import (get_numpy_arrays_with_properties, +from ._core.state import ( + get_numpy_arrays_with_properties, restore_data_arrays_with_properties) from ._components import ( PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, From 0699e1e0d30b925595c1f85b38aac1c86c9d118a Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Tue, 27 Mar 2018 15:43:02 -0700 Subject: [PATCH 32/98] Added more checks for property consistency on component creation Tests for dims and units now take place during component __init__, and raise InvalidPropertyDictError if there are issues. --- sympl/_core/base_components.py | 59 ++++++++++++++- tests/test_base_components.py | 133 +++++++++++++++++++++++++++++++++ tests/test_composite.py | 4 +- 3 files changed, 190 insertions(+), 6 deletions(-) diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index f19a039..89f024a 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -11,8 +11,33 @@ def apply_scale_factors(array_state, scale_factors): array_state[key] *= factor +class InputMixin(object): + + def __init__(self): + for name, properties in self.input_properties.items(): + if 'units' not in properties.keys(): + raise InvalidPropertyDictError( + 'Input properties do not have units defined for {}'.format(name)) + if 'dims' not in properties.keys(): + raise InvalidPropertyDictError( + 'Input properties do not have dims defined for {}'.format(name) + ) + super(InputMixin, self).__init__() + + class TendencyMixin(object): + def __init__(self): + for name, properties in self.tendency_properties.items(): + if 'units' not in properties.keys(): + raise InvalidPropertyDictError( + 'Tendency properties do not have units defined for {}'.format(name)) + if 'dims' not in properties.keys() and name not in self.input_properties.keys(): + raise InvalidPropertyDictError( + 'Tendency properties do not have dims defined for {}'.format(name) + ) + super(TendencyMixin, self).__init__() + @property def _wanted_tendency_aliases(self): wanted_tendency_aliases = {} @@ -55,6 +80,17 @@ def _check_tendencies(self, tendency_dict): class DiagnosticMixin(object): + def __init__(self): + for name, properties in self.diagnostic_properties.items(): + if 'units' not in properties.keys(): + raise InvalidPropertyDictError( + 'Diagnostic properties do not have units defined for {}'.format(name)) + if 'dims' not in properties.keys() and name not in self.input_properties.keys(): + raise InvalidPropertyDictError( + 'Diagnostic properties do not have dims defined for {}'.format(name) + ) + super(DiagnosticMixin, self).__init__() + @property def _wanted_diagnostic_aliases(self): wanted_diagnostic_aliases = {} @@ -97,6 +133,17 @@ def _check_diagnostics(self, diagnostics_dict): class OutputMixin(object): + def __init__(self): + for name, properties in self.output_properties.items(): + if 'units' not in properties.keys(): + raise InvalidPropertyDictError( + 'Output properties do not have units defined for {}'.format(name)) + if 'dims' not in properties.keys() and name not in self.input_properties.keys(): + raise InvalidPropertyDictError( + 'Output properties do not have dims defined for {}'.format(name) + ) + super(OutputMixin, self).__init__() + @property def _wanted_output_aliases(self): wanted_output_aliases = {} @@ -139,7 +186,7 @@ def _check_outputs(self, output_dict): self._check_extra_outputs(output_dict) -class Implicit(DiagnosticMixin, OutputMixin): +class Implicit(DiagnosticMixin, OutputMixin, InputMixin): """ Attributes ---------- @@ -282,6 +329,7 @@ def __init__( self._added_tendency_properties = self._insert_tendency_properties() else: self._added_tendency_properties = set() + super(Implicit, self).__init__() def _insert_tendency_properties(self): added_names = [] @@ -427,7 +475,7 @@ def array_call(self, state, timestep): pass -class Prognostic(DiagnosticMixin, TendencyMixin): +class Prognostic(DiagnosticMixin, TendencyMixin, InputMixin): """ Attributes ---------- @@ -537,6 +585,7 @@ def __init__( self.diagnostic_scale_factors = {} self.update_interval = update_interval self._last_update_time = None + super(Prognostic, self).__init__() def __call__(self, state): """ @@ -614,7 +663,7 @@ def array_call(self, state): pass -class ImplicitPrognostic(DiagnosticMixin, TendencyMixin): +class ImplicitPrognostic(DiagnosticMixin, TendencyMixin, InputMixin): """ Attributes ---------- @@ -737,6 +786,7 @@ def __init__( self.name = self.__class__.__name__.lower() else: self.name = name + super(ImplicitPrognostic, self).__init__() def __call__(self, state, timestep): """ @@ -817,7 +867,7 @@ def array_call(self, state, timestep): """ -class Diagnostic(DiagnosticMixin): +class Diagnostic(DiagnosticMixin, InputMixin): """ Attributes ---------- @@ -905,6 +955,7 @@ def __init__( self.update_interval = update_interval self._last_update_time = None self._diagnostics = None + super(Diagnostic, self).__init__() def __call__(self, state): """ diff --git a/tests/test_base_components.py b/tests/test_base_components.py index 26f2eee..704328a 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -135,6 +135,57 @@ def test_empty_prognostic(self): assert prognostic.state_given['time'] == timedelta(seconds=0) assert prognostic.times_called == 1 + def test_input_requires_dims(self): + input_properties = {'input1': {'units': 'm'}} + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + + def test_input_requires_units(self): + input_properties = {'input1': {'dims': ['dim1']}} + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + + def test_diagnostic_requires_units(self): + input_properties = {} + diagnostic_properties = {'diag1': {}} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + + def test_tendency_requires_units(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {'tend1': {}} + diagnostic_output = {} + tendency_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + def test_raises_when_tendency_not_given(self): input_properties = {} diagnostic_properties = {} @@ -808,6 +859,36 @@ def test_empty_diagnostic(self): assert diagnostic.state_given['time'] == timedelta(seconds=0) assert diagnostic.times_called == 1 + def test_input_requires_dims(self): + input_properties = {'input1': {'units': 'm'}} + diagnostic_properties = {} + diagnostic_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + diagnostic_output + ) + + def test_input_requires_units(self): + input_properties = {'input1': {'dims': ['dim1']}} + diagnostic_properties = {} + diagnostic_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + diagnostic_output + ) + + def test_diagnostic_requires_units(self): + input_properties = {} + diagnostic_properties = {'diag1': {}} + diagnostic_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + diagnostic_output, + ) + def test_raises_when_diagnostic_not_given(self): input_properties = {} diagnostic_properties = { @@ -1232,6 +1313,58 @@ def test_empty_implicit(self): assert implicit.state_given['time'] == timedelta(seconds=0) assert implicit.times_called == 1 + def test_input_requires_dims(self): + input_properties = {'input1': {'units': 'm'}} + diagnostic_properties = {} + output_properties = {} + diagnostic_output = {} + state_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, state_output + ) + + def test_input_requires_units(self): + input_properties = {'input1': {'dims': ['dim1']}} + diagnostic_properties = {} + output_properties = {} + diagnostic_output = {} + state_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, state_output + ) + + def test_diagnostic_requires_units(self): + input_properties = {} + diagnostic_properties = {'diag1': {}} + output_properties = {} + diagnostic_output = {} + state_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, state_output + ) + + def test_output_requires_units(self): + input_properties = {} + diagnostic_properties = {} + output_properties = {'output1': {}} + diagnostic_output = {} + state_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, state_output + ) + def test_timedelta_is_passed(self): implicit = MockImplicit({}, {}, {}, {}, {}) tendencies, diagnostics = implicit( diff --git a/tests/test_composite.py b/tests/test_composite.py index 6d29218..b320a7f 100644 --- a/tests/test_composite.py +++ b/tests/test_composite.py @@ -336,9 +336,9 @@ def test_diagnostic_composite_single_component_missing_dims_on_diagnostic(): }, } diagnostic_output = {} - diagnostic = MockDiagnostic( - input_properties, diagnostic_properties, diagnostic_output) try: + diagnostic = MockDiagnostic( + input_properties, diagnostic_properties, diagnostic_output) DiagnosticComposite(diagnostic) except InvalidPropertyDictError: pass From 08199304d9c189828bc18417737031fcf1fdf173 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Tue, 27 Mar 2018 15:46:20 -0700 Subject: [PATCH 33/98] Fixed dev requirements version of xarray --- requirements_dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index a831e4c..1727b5f 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -11,7 +11,7 @@ PyYAML==3.11 pytest==2.9.2 pytest-cov==2.4.0 mock==2.0.0 -xarray==0.8.2 +xarray>=0.9.3 six pint==0.7.0 scipy From 614069f937cde09f0cf1168ba7bb39d7b12b17bc Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 28 Mar 2018 11:45:32 -0700 Subject: [PATCH 34/98] Added tendencies_in_diagnostics option to Prognostic Users will want to separate tendencies by source component, so they should be able to use this option either on TimeSteppers or on Prognostics. --- sympl/_core/base_components.py | 157 ++++++++++++++++++++++++++--- sympl/_core/state.py | 11 ++- tests/test_base_components.py | 176 +++++++++++++++++++++++++++++++++ 3 files changed, 326 insertions(+), 18 deletions(-) diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 89f024a..b8c73f9 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -3,7 +3,7 @@ from .time import timedelta from .exceptions import ( InvalidPropertyDictError, ComponentExtraOutputError, - ComponentMissingOutputError) + ComponentMissingOutputError, InvalidStateError) def apply_scale_factors(array_state, scale_factors): @@ -24,6 +24,11 @@ def __init__(self): ) super(InputMixin, self).__init__() + def _check_inputs(self, state): + for key in self.input_properties.keys(): + if key not in state.keys(): + raise InvalidStateError('Missing input quantity {}'.format(key)) + class TendencyMixin(object): @@ -111,7 +116,7 @@ def _check_missing_diagnostics(self, diagnostics_dict): missing_diagnostics.add(name) if len(missing_diagnostics) > 0: raise ComponentMissingOutputError( - 'Component {} did not compute diagnostics {}'.format( + 'Component {} did not compute diagnostic(s) {}'.format( self.__class__.__name__, ', '.join(missing_diagnostics))) def _check_extra_diagnostics(self, diagnostics_dict): @@ -122,7 +127,7 @@ def _check_extra_diagnostics(self, diagnostics_dict): extra_diagnostics = set(diagnostics_dict.keys()).difference(wanted_set) if len(extra_diagnostics) > 0: raise ComponentExtraOutputError( - 'Component {} computed diagnostics {} which are not in ' + 'Component {} computed diagnostic(s) {} which are not in ' 'diagnostic_properties'.format( self.__class__.__name__, ', '.join(extra_diagnostics))) @@ -166,7 +171,7 @@ def _check_missing_outputs(self, outputs_dict): missing_outputs.add(name) if len(missing_outputs) > 0: raise ComponentMissingOutputError( - 'Component {} did not compute outputs {}'.format( + 'Component {} did not compute output(s) {}'.format( self.__class__.__name__, ', '.join(missing_outputs))) def _check_extra_outputs(self, outputs_dict): @@ -177,7 +182,7 @@ def _check_extra_outputs(self, outputs_dict): extra_outputs = set(outputs_dict.keys()).difference(wanted_set) if len(extra_outputs) > 0: raise ComponentExtraOutputError( - 'Component {} computed outputs {} which are not in ' + 'Component {} computed output(s) {} which are not in ' 'output_properties'.format( self.__class__.__name__, ', '.join(extra_outputs))) @@ -339,16 +344,20 @@ def _insert_tendency_properties(self): units = 's^-1' else: units = '{} s^-1'.format(properties['units']) + if 'dims' in properties.keys(): + dims = properties['dims'] + else: + dims = self.input_properties[name]['dims'] self.diagnostic_properties[tendency_name] = { 'units': units, - 'dims': properties['dims'], + 'dims': dims, } if name not in self.input_properties.keys(): self.input_properties[name] = { - 'dims': properties['dims'], + 'dims': dims, 'units': properties['units'], } - elif self.input_properties[name]['dims'] != self.output_properties[name]['dims']: + elif self.input_properties[name]['dims'] != dims: raise InvalidPropertyDictError( 'Can only calculate tendencies when input and output values' ' have the same dimensions, but dims for {} are ' @@ -420,6 +429,7 @@ def __call__(self, state, timestep): if (self.update_interval is None or self._last_update_time is None or state['time'] >= self._last_update_time + self.update_interval): + self._check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) raw_state['time'] = state['time'] apply_scale_factors(raw_state, self.input_scale_factors) @@ -507,6 +517,13 @@ class Prognostic(DiagnosticMixin, TendencyMixin, InputMixin): If not None, the component will only give new output if at least a period of update_interval has passed since the last time new output was given. Otherwise, it would return that cached output. + tendencies_in_diagnostics : boo + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output based on first order time + differencing of output values. + name : string + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". """ __metaclass__ = abc.ABCMeta @@ -548,7 +565,8 @@ def __repr__(self): def __init__( self, input_scale_factors=None, tendency_scale_factors=None, - diagnostic_scale_factors=None, update_interval=None): + diagnostic_scale_factors=None, update_interval=None, + tendencies_in_diagnostics=False, name=None): """ Initializes the Implicit object. @@ -570,6 +588,13 @@ def __init__( If given, the component will only give new output if at least a period of update_interval has passed since the last time new output was given. Otherwise, it would return that cached output. + tendencies_in_diagnostics : bool, optional + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output. + name : string, optional + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". By default the class name in + lowercase is used. """ if input_scale_factors is not None: self.input_scale_factors = input_scale_factors @@ -585,8 +610,36 @@ def __init__( self.diagnostic_scale_factors = {} self.update_interval = update_interval self._last_update_time = None + self._tendencies_in_diagnostics = tendencies_in_diagnostics + if name is None: + self.name = self.__class__.__name__ + else: + self.name = name + self._added_diagnostic_names = [] + if self.tendencies_in_diagnostics: + self._insert_tendency_properties() super(Prognostic, self).__init__() + @property + def tendencies_in_diagnostics(self): + return self._tendencies_in_diagnostics + + def _insert_tendency_properties(self): + for name, properties in self.tendency_properties.items(): + tendency_name = self._get_tendency_name(name) + if 'dims' in properties.keys(): + dims = properties['dims'] + else: + dims = self.input_properties[name]['dims'] + self.diagnostic_properties[tendency_name] = { + 'units': properties['units'], + 'dims': dims, + } + self._added_diagnostic_names.append(tendency_name) + + def _get_tendency_name(self, name): + return '{}_tendency_from_{}'.format(name, self.name) + def __call__(self, state): """ Gets tendencies and diagnostics from the passed model state. @@ -631,10 +684,30 @@ def __call__(self, state): state, self.input_properties) self._diagnostics = restore_data_arrays_with_properties( raw_diagnostics, self.diagnostic_properties, - state, self.input_properties) + state, self.input_properties, + ignore_names=self._added_diagnostic_names) + if self.tendencies_in_diagnostics: + self._insert_tendencies_to_diagnostics() self._last_update_time = state['time'] return self._tendencies, self._diagnostics + def _insert_tendencies_to_diagnostics(self): + for name, value in self._tendencies.items(): + tendency_name = self._get_tendency_name(name) + self._diagnostics[tendency_name] = value + + def _check_missing_diagnostics(self, diagnostics_dict): + missing_diagnostics = set() + for name, aliases in self._wanted_diagnostic_aliases.items(): + if (name not in diagnostics_dict.keys() and + name not in self._added_diagnostic_names and + not any(alias in diagnostics_dict.keys() for alias in aliases)): + missing_diagnostics.add(name) + if len(missing_diagnostics) > 0: + raise ComponentMissingOutputError( + 'Component {} did not compute diagnostics {}'.format( + self.__class__.__name__, ', '.join(missing_diagnostics))) + @abc.abstractmethod def array_call(self, state): """ @@ -699,6 +772,13 @@ class ImplicitPrognostic(DiagnosticMixin, TendencyMixin, InputMixin): A label to be used for this object, for example as would be used for Y in the name "X_tendency_from_Y". By default the class name in lowercase is used. + tendencies_in_diagnostics : boo + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output based on first order time + differencing of output values. + name : string + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". """ __metaclass__ = abc.ABCMeta @@ -738,7 +818,8 @@ def __repr__(self): def __init__( self, input_scale_factors=None, tendency_scale_factors=None, - diagnostic_scale_factors=None, update_interval=None, name=None): + diagnostic_scale_factors=None, update_interval=None, + tendencies_in_diagnostics=False, name=None): """ Initializes the Implicit object. @@ -756,13 +837,13 @@ def __init__( A (possibly empty) dictionary whose keys are quantity names and values are floats by which diagnostic values are scaled before being returned by this object. - tendencies_in_diagnostics : bool, optional - A boolean indicating whether this object will put tendencies of - quantities in its diagnostic output. update_interval : timedelta, optional If given, the component will only give new output if at least a period of update_interval has passed since the last time new output was given. Otherwise, it would return that cached output. + tendencies_in_diagnostics : bool, optional + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output. name : string, optional A label to be used for this object, for example as would be used for Y in the name "X_tendency_from_Y". By default the class name in @@ -782,12 +863,36 @@ def __init__( self.diagnostic_scale_factors = {} self.update_interval = update_interval self._last_update_time = None + self._tendencies_in_diagnostics = tendencies_in_diagnostics if name is None: - self.name = self.__class__.__name__.lower() + self.name = self.__class__.__name__ else: self.name = name + self._added_diagnostic_names = [] + if self.tendencies_in_diagnostics: + self._insert_tendency_properties() super(ImplicitPrognostic, self).__init__() + @property + def tendencies_in_diagnostics(self): + return self._tendencies_in_diagnostics + + def _insert_tendency_properties(self): + for name, properties in self.tendency_properties.items(): + tendency_name = self._get_tendency_name(name) + if 'dims' in properties.keys(): + dims = properties['dims'] + else: + dims = self.input_properties[name]['dims'] + self.diagnostic_properties[tendency_name] = { + 'units': properties['units'], + 'dims': dims, + } + self._added_diagnostic_names.append(tendency_name) + + def _get_tendency_name(self, name): + return '{}_tendency_from_{}'.format(name, self.name) + def __call__(self, state, timestep): """ Gets tendencies and diagnostics from the passed model state. @@ -834,10 +939,30 @@ def __call__(self, state, timestep): state, self.input_properties) self._diagnostics = restore_data_arrays_with_properties( raw_diagnostics, self.diagnostic_properties, - state, self.input_properties) + state, self.input_properties, + ignore_names=self._added_diagnostic_names) + if self.tendencies_in_diagnostics: + self._insert_tendencies_to_diagnostics() self._last_update_time = state['time'] return self._tendencies, self._diagnostics + def _insert_tendencies_to_diagnostics(self): + for name, value in self._tendencies.items(): + tendency_name = self._get_tendency_name(name) + self._diagnostics[tendency_name] = value + + def _check_missing_diagnostics(self, diagnostics_dict): + missing_diagnostics = set() + for name, aliases in self._wanted_diagnostic_aliases.items(): + if (name not in diagnostics_dict.keys() and + name not in self._added_diagnostic_names and + not any(alias in diagnostics_dict.keys() for alias in aliases)): + missing_diagnostics.add(name) + if len(missing_diagnostics) > 0: + raise ComponentMissingOutputError( + 'Component {} did not compute diagnostics {}'.format( + self.__class__.__name__, ', '.join(missing_diagnostics))) + @abc.abstractmethod def array_call(self, state, timestep): """ diff --git a/sympl/_core/state.py b/sympl/_core/state.py index 05a5ec4..85d1079 100644 --- a/sympl/_core/state.py +++ b/sympl/_core/state.py @@ -143,7 +143,10 @@ def get_numpy_array(data_array, out_dims, dim_lengths): def restore_data_arrays_with_properties( - raw_arrays, output_properties, input_state, input_properties): + raw_arrays, output_properties, input_state, input_properties, + ignore_names=None): + if ignore_names is None: + ignore_names = [] wildcard_names, dim_lengths = get_wildcard_matches_and_dim_lengths( input_state, input_properties) for name, value in raw_arrays.items(): @@ -151,7 +154,9 @@ def restore_data_arrays_with_properties( raw_arrays[name] = np.asarray(value) out_dims_property = {} for name, properties in output_properties.items(): - if 'dims' in properties.keys(): + if name in ignore_names: + continue + elif 'dims' in properties.keys(): out_dims_property[name] = properties['dims'] elif name not in input_properties.keys(): raise InvalidPropertyDictError( @@ -163,6 +168,8 @@ def restore_data_arrays_with_properties( out_dims_property[name] = input_properties[name]['dims'] out_dict = {} for name, dims in out_dims_property.items(): + if name in ignore_names: + continue if 'alias' in output_properties[name].keys(): raw_name = output_properties[name]['alias'] elif name in input_properties.keys() and 'alias' in input_properties[name].keys(): diff --git a/tests/test_base_components.py b/tests/test_base_components.py index 704328a..a1f60f5 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -825,6 +825,135 @@ def test_update_interval_on_datetime(self): prognostic, {'time': dt + timedelta(seconds=50)}) assert prognostic.times_called == 5, 'should not re-compute output' + def test_tendencies_in_diagnostics_no_tendency(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, tendencies_in_diagnostics=True + ) + assert prognostic.input_properties == {} + assert prognostic.diagnostic_properties == {} + assert prognostic.tendency_properties == {} + state = {'time': timedelta(0)} + _, diagnostics = self.call_component(prognostic, state) + assert diagnostics == {} + + def test_tendencies_in_diagnostics_one_tendency(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + } + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 20., + } + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, tendencies_in_diagnostics=True, + ) + tendency_name = 'output1_tendency_from_{}'.format(prognostic.__class__.__name__) + assert len(prognostic.diagnostic_properties) == 1 + assert tendency_name in prognostic.diagnostic_properties.keys() + properties = prognostic.diagnostic_properties[tendency_name] + assert properties['dims'] == ['dim1'] + assert properties['units'] == 'm/s' + state = { + 'time': timedelta(0), + } + _, diagnostics = self.call_component(prognostic, state) + assert tendency_name in diagnostics.keys() + assert len( + diagnostics[tendency_name].dims) == 1 + assert 'dim1' in diagnostics[tendency_name].dims + assert diagnostics[tendency_name].attrs['units'] == 'm/s' + assert np.all(diagnostics[tendency_name].values == 20.) + + def test_tendencies_in_diagnostics_one_tendency_dims_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'units': 'm/s' + } + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 20., + } + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, tendencies_in_diagnostics=True, + ) + tendency_name = 'output1_tendency_from_{}'.format(prognostic.__class__.__name__) + assert len(prognostic.diagnostic_properties) == 1 + assert tendency_name in prognostic.diagnostic_properties.keys() + properties = prognostic.diagnostic_properties[tendency_name] + assert properties['dims'] == ['dim1'] + assert properties['units'] == 'm/s' + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'}), + } + _, diagnostics = self.call_component(prognostic, state) + assert tendency_name in diagnostics.keys() + assert len( + diagnostics[tendency_name].dims) == 1 + assert 'dim1' in diagnostics[tendency_name].dims + assert diagnostics[tendency_name].attrs['units'] == 'm/s' + assert np.all(diagnostics[tendency_name].values == 20.) + + def test_tendencies_in_diagnostics_one_tendency_with_component_name(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + } + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 20., + } + prognostic = self.component_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, tendencies_in_diagnostics=True, + name='component', + ) + tendency_name = 'output1_tendency_from_component' + assert len(prognostic.diagnostic_properties) == 1 + assert tendency_name in prognostic.diagnostic_properties.keys() + properties = prognostic.diagnostic_properties[tendency_name] + assert properties['dims'] == ['dim1'] + assert properties['units'] == 'm/s' + state = { + 'time': timedelta(0), + } + _, diagnostics = self.call_component(prognostic, state) + print(diagnostics.keys()) + assert tendency_name in diagnostics.keys() + assert len( + diagnostics[tendency_name].dims) == 1 + assert 'dim1' in diagnostics[tendency_name].dims + assert diagnostics[tendency_name].attrs['units'] == 'm/s' + assert np.all(diagnostics[tendency_name].values == 20.) + class ImplicitPrognosticTests(PrognosticTests): @@ -2074,6 +2203,53 @@ def test_tendencies_in_diagnostics_one_tendency(self): assert np.all( diagnostics['output1_tendency_from_mockimplicit'].values == 2.) + def test_tendencies_in_diagnostics_one_tendency_dims_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + diagnostic_properties = {} + output_properties = { + 'output1': { + 'units': 'm' + } + } + diagnostic_output = {} + output_state = { + 'output1': np.ones([10]) * 20., + } + implicit = MockImplicit( + input_properties, diagnostic_properties, output_properties, + diagnostic_output, output_state, tendencies_in_diagnostics=True, + ) + assert len(implicit.diagnostic_properties) == 1 + assert 'output1_tendency_from_mockimplicit' in implicit.diagnostic_properties.keys() + assert 'output1' in input_properties.keys(), 'Implicit needs original value to calculate tendency' + assert input_properties['output1']['dims'] == ['dim1'] + assert input_properties['output1']['units'] == 'm' + properties = implicit.diagnostic_properties[ + 'output1_tendency_from_mockimplicit'] + assert properties['dims'] == ['dim1'] + assert properties['units'] == 'm s^-1' + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10])*10., + dims=['dim1'], + attrs={'units': 'm'} + ), + } + diagnostics, _ = implicit(state, timedelta(seconds=5)) + assert 'output1_tendency_from_mockimplicit' in diagnostics.keys() + assert len( + diagnostics['output1_tendency_from_mockimplicit'].dims) == 1 + assert 'dim1' in diagnostics['output1_tendency_from_mockimplicit'].dims + assert diagnostics['output1_tendency_from_mockimplicit'].attrs['units'] == 'm s^-1' + assert np.all( + diagnostics['output1_tendency_from_mockimplicit'].values == 2.) + def test_tendencies_in_diagnostics_one_tendency_mismatched_units(self): input_properties = { 'output1': { From e71ea5330e7cb64afe5c49669156a090c7dc8a4c Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 28 Mar 2018 11:46:25 -0700 Subject: [PATCH 35/98] Removed unused util functions --- sympl/_core/util.py | 267 -------------------------------------------- 1 file changed, 267 deletions(-) diff --git a/sympl/_core/util.py b/sympl/_core/util.py index e149631..d807f18 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -30,161 +30,6 @@ class ShapeMismatchError(Exception): pass -def ensure_consistent_dimension_lengths(state): - dimension_lengths = {} - for name, array in state.items(): - if isinstance(array, DataArray): - for i, name in enumerate(array.dims): - if name in dimension_lengths and (array.shape[i] != dimension_lengths[name]): - raise InvalidStateError( - 'dimension {} has multiple lengths (at least {} and {})'.format( - name, array.shape[i], dimension_lengths[name])) - else: - dimension_lengths[name] = array.shape[i] - - -def ensure_properties_have_dims_and_units(properties, quantity_name): - if 'dims' not in properties: - raise InvalidPropertyDictError( - 'dims not specified for quantity {}'.format(quantity_name)) - if 'units' not in properties: - raise InvalidPropertyDictError( - 'units not specified for quantity {}'.format(quantity_name)) - - -def ensure_quantity_has_units(quantity, quantity_name): - if 'units' not in quantity.attrs: - raise InvalidStateError( - 'quantity {} is missing units attribute'.format(quantity_name)) - - -def independent_wildcards_first(item_list): - """ - Sorts the items in a (quantity_name, properties) item list so that - quantities that match_dims_like other quantities are placed *after* those - quantities in the list. In the case that two quantities match_dims_like - one another, a InvalidPropertyDictError is raised. - - This is necessary so that information about the wildcard matches from - previous calls can be given to later calls to ensure wildcard-matched - dimensions are ordered the same. - """ - name_list = [] - properties_list = [] - matched_by_dict = {} - for quantity_name, properties in item_list: - if 'match_dims_like' in properties: - target_name = properties['match_dims_like'] - if target_name in matched_by_dict: - matched_by_dict[target_name].append(quantity_name) - else: - matched_by_dict[target_name] = [quantity_name] - for quantity_name, properties in item_list: - if quantity_name not in matched_by_dict: - name_list.append(quantity_name) - properties_list.append(properties) - else: - matched_by_indices = [] - for matched_by_name in matched_by_dict[quantity_name]: - if matched_by_name in name_list: - matched_by_indices.append(name_list.index(matched_by_name)) - if len(matched_by_indices) == 0: - name_list.append(quantity_name) - properties_list.append(properties) - else: - index = min(matched_by_indices) - name_list.insert(index, quantity_name) - properties_list.insert(index, properties) - return zip(name_list, properties_list) - - -def get_numpy_arrays_with_properties(state, property_dictionary): - """ - Parameters - ---------- - state : dict - A state dictionary. - property_dictionary : dict - A dictionary whose keys are quantity names and values are dictionaries - with properties for those quantities. The property "dims" must be - present, indicating the dimensions that the quantity must have when it - is returned as a numpy array. The property "units" must be present, - and will be used to check the units on the input state and perform - a conversion if necessary. If the optional property "match_dims_like" - is present, its value should be a quantity also present in - property_dictionary, and it will be ensured that any shared wildcard - dimensions ('x', 'y', 'z', '*') for this quantity match the same - dimensions as the specified quantity. - - Returns - ------- - out_dict : dict - A dictionary whose keys are quantity names and values are numpy arrays - containing the data for those quantities, as specified by - property_dictionary. - - Raises - ------ - InvalidStateError - If a DataArray in the state is missing an explicitly-specified - dimension defined in its properties (dimension names other than - 'x', 'y', 'z', or '*'), or if the state is missing a required quantity. - InvalidPropertyError - If a quantity in property_dictionary is missing values for "dims" or - "units". - """ - ensure_consistent_dimension_lengths(state) - out_dict = {} - matches = {} - for quantity_name, properties in independent_wildcards_first(property_dictionary.items()): - ensure_properties_have_dims_and_units(properties, quantity_name) - if quantity_name not in state.keys(): - raise InvalidStateError( - 'state is missing quantity {}'.format(quantity_name)) - ensure_quantity_has_units(state[quantity_name], quantity_name) - quantity_has_alias = 'alias' in properties.keys() - if quantity_has_alias: - out_name = properties['alias'] - else: - out_name = quantity_name - if out_name in out_dict.keys(): - raise InvalidPropertyDictError( - 'Multiple arrays with output name {}'.format(out_name)) - try: - quantity_array = state[quantity_name].to_units(properties['units']) - except ValueError: - raise ValueError( - 'Invalid target units {} for quantity {} ' - 'with units {}'.format( - properties['units'], - quantity_name, - state[quantity_name].attrs['units'])) - try: - if ('match_dims_like' in properties.keys() and - properties['match_dims_like'] in matches): - out_dict[out_name], matches[quantity_name] = get_numpy_array( - quantity_array, - out_dims=properties['dims'], return_wildcard_matches=True, - require_wildcard_matches=matches[properties['match_dims_like']]) - else: - out_dict[out_name], matches[quantity_name] = get_numpy_array( - quantity_array, - out_dims=properties['dims'], return_wildcard_matches=True) - except NoMatchForDirectionError as err: - raise InvalidStateError( - 'dimension {} is missing from quantity {}'.format( - err, quantity_name) - ) - except DimensionNotInOutDimsError as err: - raise InvalidStateError( - 'dims property {} on quantity {} does not allow for state' - 'quantity to have dimension {} (but it does)'.format( - properties['dims'], quantity_name, err) - ) - ensure_dims_like_are_satisfied(matches, property_dictionary) - return out_dict - - def get_numpy_array( data_array, out_dims, return_wildcard_matches=False, require_wildcard_matches=None): @@ -270,118 +115,6 @@ def get_numpy_array( return return_array -def ensure_dims_like_are_satisfied(matches, property_dictionary): - for quantity_name, properties in property_dictionary.items(): - if 'match_dims_like' in properties: - if properties['match_dims_like'] not in property_dictionary.keys(): - raise InvalidPropertyDictError( - 'quantity {} is not specified in property dictionary, ' - 'but is referred to by {} in match_dims_like'.format( - properties['match_dims_like'], quantity_name - )) - like_name = properties['match_dims_like'] - for wildcard_dim in set(matches[quantity_name].keys()).intersection( - matches[like_name].keys()): - # We must use == because we need the dim order to be the same - if not same_list( - matches[quantity_name][wildcard_dim], - matches[like_name][wildcard_dim]): - raise InvalidStateError( - 'quantity {} matches dimensions {} for direction {}, but ' - 'is referred to in match_dims_like by quantity {} with matches ' - '{}'.format( - like_name, matches[like_name][wildcard_dim], - wildcard_dim, quantity_name, - matches[quantity_name][wildcard_dim])) - - -def restore_data_arrays_with_properties( - raw_arrays, output_properties, input_state, input_properties): - """ - Parameters - ---------- - raw_arrays : dict - A dictionary whose keys are quantity names and values are numpy arrays - containing the data for those quantities. - output_properties : dict - A dictionary whose keys are quantity names and values are dictionaries - with properties for those quantities. The property "dims_like" must be - present, and specifies an input quantity that the dimensions of the - output quantity should be like. All other properties are included as - attributes on the output DataArray for that quantity, including "units" - which is required. - input_state : dict - A state dictionary that was used as input to a component for which - DataArrays are being restored. - input_properties : dict - A dictionary whose keys are quantity names and values are dictionaries - with input properties for those quantities. The property "dims" must be - present, indicating the dimensions that the quantity was transformed to - when taken as input to a component. - - Returns - ------- - out_dict : dict - A dictionary whose keys are quantities and values are DataArrays - corresponding to those quantities, with data, shapes and attributes - determined from the inputs to this function. - - Raises - ------ - InvalidPropertyDictError - When an output property is specified to have dims_like an input - property, but the arrays for the two properties have incompatible - shapes. - """ - ensure_consistent_dimension_lengths(input_state) - out_dict = {} - for quantity_name, properties in output_properties.items(): - attrs = properties.copy() - dims_like = attrs.pop('dims_like', quantity_name) - if (quantity_name not in raw_arrays.keys()) and ('alias' in properties): - from_name = attrs.pop('alias') - elif quantity_name in input_properties.keys() and 'alias' in input_properties[quantity_name].keys(): - from_name = input_properties[quantity_name]['alias'] - else: - from_name = quantity_name - if from_name not in raw_arrays.keys(): - raise ValueError( - 'requested output {} is not present in raw_arrays'.format( - from_name)) - array = raw_arrays[from_name] - if dims_like in input_properties.keys(): - from_dims = input_properties[dims_like]['dims'] - result_like = input_state[dims_like] - try: - out_dict[quantity_name] = restore_dimensions( - array, - from_dims=from_dims, - result_like=result_like, - result_attrs=attrs) - except ShapeMismatchError: - raise InvalidPropertyDictError( - 'output quantity {} has dims_like input {}, but the ' - 'provided output array for {} has ' - 'a shape {} incompatible with the input shape {} of {}. ' - 'Do they really have the same dimensions?'.format( - quantity_name, dims_like, quantity_name, array.shape, - result_like.shape, dims_like - ) - ) - elif 'dims' in properties.keys(): - out_dict[quantity_name] = DataArray( - array, - dims=properties['dims'], - attrs={'units': properties['units']}, - ) - else: - raise InvalidPropertyDictError( - 'Could not determine dimensions for {}, make sure dims_like or ' - 'dims is specified in its property dictionary'.format( - quantity_name)) - return out_dict - - def restore_dimensions(array, from_dims, result_like, result_attrs=None): """ Restores a numpy array to a DataArray with similar dimensions to a reference From 723656c4f596a46ca709ac73c8d53cefcd7e97b1 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 28 Mar 2018 13:09:35 -0700 Subject: [PATCH 36/98] Removed unused import --- sympl/_core/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sympl/_core/util.py b/sympl/_core/util.py index d807f18..f3dda66 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -3,7 +3,7 @@ from .units import units_are_compatible from .array import DataArray from .exceptions import ( - SharedKeyError, InvalidStateError, InvalidPropertyDictError) + SharedKeyError, InvalidPropertyDictError) try: from numba import jit From 95025dbf7a8c2a782de395a1b1a4632f0da21c3e Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 28 Mar 2018 13:32:04 -0700 Subject: [PATCH 37/98] Fixed a bug that led to TimeSteppers modifying tendencies returned by Prognostics --- HISTORY.rst | 2 ++ sympl/_core/util.py | 5 ++++- tests/test_timestepping.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index c7020f6..3214609 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -29,6 +29,8 @@ Latest respectively if outputs do not match. * Added a priority order of property types for determining which aliases are returned by get_component_aliases +* Fixed a bug where TimeStepper objects would modify the arrays passed to them by + Prognostic objects, leading to unexpected value changes. Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/_core/util.py b/sympl/_core/util.py index f3dda66..d78ec00 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -252,7 +252,10 @@ def update_dict_by_adding_another(dict1, dict2): """ for key in dict2.keys(): if key not in dict1: - dict1[key] = dict2[key] + if hasattr(dict2[key], 'copy'): + dict1[key] = dict2[key].copy() + else: + dict1[key] = dict2[key] else: if (isinstance(dict1[key], DataArray) and isinstance(dict2[key], DataArray) and ('units' in dict1[key].attrs) and ('units' in dict2[key].attrs)): diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index def5114..e310ea9 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -333,6 +333,44 @@ def test_dataarray_three_steps(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' + def test_given_tendency_not_modified_with_two_components(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'tend1': { + 'dims': ['dim1'], + 'units': 'm/s', + } + } + diagnostic_output = {} + tendency_output_1 = { + 'tend1': np.ones([10]) * 5. + } + prognostic1 = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output_1 + ) + tendency_output_2 = { + 'tend1': np.ones([10]) * 5. + } + prognostic2 = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output_2 + ) + stepper = self.timestepper_class( + prognostic1, prognostic2, tendencies_in_diagnostics=True) + state = { + 'time': timedelta(0), + 'tend1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'}, + ) + } + _, _ = stepper(state, timedelta(seconds=5)) + assert np.all(tendency_output_1['tend1'] == 5.) + assert np.all(tendency_output_2['tend1'] == 5.) + def test_tendencies_in_diagnostics_no_tendency(self): input_properties = {} diagnostic_properties = {} From 2b133820ef3239c11d47247311ee6e0012563d20 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 28 Mar 2018 13:55:27 -0700 Subject: [PATCH 38/98] Fixed a bug where get_constants_string missed some constants Certain condensible related constants were missing, and user defined constants were missing. Closed #27. --- HISTORY.rst | 2 ++ sympl/_core/constants.py | 49 +++++++++++++++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3214609..f8f368f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -31,6 +31,8 @@ Latest returned by get_component_aliases * Fixed a bug where TimeStepper objects would modify the arrays passed to them by Prognostic objects, leading to unexpected value changes. +* Fixed a bug where constants were missing from the string returned by + get_constants_string, particularly any new constants (issue #27) Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/_core/constants.py b/sympl/_core/constants.py index 3fc1dfb..040f3f0 100644 --- a/sympl/_core/constants.py +++ b/sympl/_core/constants.py @@ -9,10 +9,12 @@ def __repr__(self): def _repr(self, sphinx=False): return_string = '' + printed_names = set() for category, name_list in constant_names_by_category.items(): if len(name_list) > 0: return_string += category.title() + '\n' for name in name_list: + printed_names.add(name) units = self[name].attrs['units'] units = units.replace('dimensionless', '') return_string += '\t{}: {} {}\n'.format( @@ -20,6 +22,17 @@ def _repr(self, sphinx=False): if sphinx: return_string += '\n' return_string += '\n' + print(set(self.keys()).difference(printed_names)) + if len(set(self.keys()).difference(printed_names)) > 0: + return_string += 'User Defined\n' + for name in self.keys(): + if name not in printed_names: + units = self[name].attrs['units'] + return_string += '\t{}: {} {}\n'.format( + name, self[name].values.item(), units) + if sphinx: + return_string += '\n' + return_string += '\n' return return_string @property @@ -120,7 +133,8 @@ def __getitem__(self, item): 'gravitational_acceleration', 'planetary_radius', 'planetary_rotation_rate', - 'seconds_per_day'], + 'seconds_per_day' + ], 'physical': [ 'stefan_boltzmann_constant', @@ -129,7 +143,8 @@ def __getitem__(self, item): 'boltzmann_constant', 'loschmidt_constant', 'universal_gas_constant', - 'planck_constant'], + 'planck_constant' + ], 'condensible': [ 'density_of_liquid_phase', @@ -146,7 +161,10 @@ def __getitem__(self, item): 'thermal_conductivity_of_solid_phase_as_ice', 'thermal_conductivity_of_solid_phase_as_snow', 'thermal_conductivity_of_liquid_phase', - 'freezing_temperature_of_liquid_phase'], + 'freezing_temperature_of_liquid_phase', + 'enthalpy_of_fusion', + 'latent_heat_of_vaporization', + ], 'atmospheric': [ 'heat_capacity_of_dry_air_at_constant_pressure', @@ -155,9 +173,12 @@ def __getitem__(self, item): 'reference_air_pressure'], 'stellar': [ - 'stellar_irradiance'], + 'stellar_irradiance', + 'solar_constant', + ], - 'oceanographic': [], + 'oceanographic': [ + ], 'chemical': [ 'heat_capacity_of_water_vapor_at_constant_pressure', @@ -165,7 +186,23 @@ def __getitem__(self, item): 'gas_constant_of_water_vapor', 'latent_heat_of_vaporization_of_water', 'heat_capacity_of_liquid_water', - 'latent_heat_of_fusion_of_water'], + 'latent_heat_of_fusion_of_water', + 'heat_capacity_of_solid_water_as_ice', + 'heat_capacity_of_solid_water_as_snow', + 'thermal_conductivity_of_solid_water_as_ice', + 'thermal_conductivity_of_solid_water_as_snow', + 'thermal_conductivity_of_liquid_water', + 'density_of_solid_water_as_ice', + 'density_of_solid_water_as_snow', + 'freezing_temperature_of_liquid_water', + 'specific_enthalpy_of_water_vapor', + 'density_of_snow', + 'heat_capacity_of_snow', + 'heat_capacity_of_ice', + 'density_of_ice', + 'thermal_conductivity_of_ice', + 'thermal_conductivity_of_snow', + ], } From 0207fe16fc35f12117e70ab2abec0651be4894a0 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 28 Mar 2018 17:17:17 -0700 Subject: [PATCH 39/98] Fixed basic components to call superclasses. --- sympl/_components/basic.py | 165 +++++++++++++++++++++++++++++++++++-- 1 file changed, 156 insertions(+), 9 deletions(-) diff --git a/sympl/_components/basic.py b/sympl/_components/basic.py index da6af52..f249133 100644 --- a/sympl/_components/basic.py +++ b/sympl/_components/basic.py @@ -7,9 +7,49 @@ class ConstantPrognostic(Prognostic): """ Prescribes constant tendencies provided at initialization. - Note: Any arrays in the passed dictionaries are not copied, so that - if you were to modify them after passing them into this object, - it would also modify the values inside this object. + Attributes + ---------- + input_properties : dict + A dictionary whose keys are quantities required in the state when the + object is called, and values are dictionaries which indicate 'dims' and + 'units'. + tendency_properties : dict + A dictionary whose keys are quantities for which tendencies are returned when the + object is called, and values are dictionaries which indicate 'dims' and + 'units'. + diagnostic_properties : dict + A dictionary whose keys are diagnostic quantities returned when the + object is called, and values are dictionaries which indicate 'dims' and + 'units'. + input_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + tendency_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which tendency values are scaled before being + returned by this object. + diagnostic_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + update_interval : timedelta + If not None, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. + tendencies_in_diagnostics : boo + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output based on first order time + differencing of output values. + name : string + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". + + Note + ---- + Any arrays in the passed dictionaries are not copied, so that + if you were to modify them after passing them into this object, + it would also modify the values inside this object. """ @property @@ -36,7 +76,7 @@ def diagnostic_properties(self): } return return_dict - def __init__(self, tendencies, diagnostics=None): + def __init__(self, tendencies, diagnostics=None, **kwargs): """ Args ---- @@ -44,17 +84,41 @@ def __init__(self, tendencies, diagnostics=None): A dictionary whose keys are strings indicating state quantities and values are the time derivative of those quantities in units/second to be returned by this Prognostic. - diagnostics : dict + diagnostics : dict, optional A dictionary whose keys are strings indicating state quantities and values are the value of those quantities - to be returned by this Prognostic. + to be returned by this Prognostic. By default an empty dictionary + is used. + input_scale_factors : dict, optional + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + tendency_scale_factors : dict, optional + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which tendency values are scaled before being + returned by this object. + diagnostic_scale_factors : dict, optional + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + update_interval : timedelta, optional + If given, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. + tendencies_in_diagnostics : bool, optional + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output. + name : string, optional + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". By default the class name in + lowercase is used. """ self.__tendencies = tendencies.copy() if diagnostics is not None: self.__diagnostics = diagnostics.copy() else: self.__diagnostics = {} - super(ConstantPrognostic, self).__init__() + super(ConstantPrognostic, self).__init__(**kwargs) def array_call(self, state): tendencies = {} @@ -70,6 +134,29 @@ class ConstantDiagnostic(Diagnostic): """ Yields constant diagnostics provided at initialization. + Attributes + ---------- + input_properties : dict + A dictionary whose keys are quantities required in the state when the + object is called, and values are dictionaries which indicate 'dims' and + 'units'. + diagnostic_properties : dict + A dictionary whose keys are diagnostic quantities returned when the + object is called, and values are dictionaries which indicate 'dims' and + 'units'. + input_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + diagnostic_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + update_interval : timedelta + If not None, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. + Note ---- Any arrays in the passed dictionaries are not copied, so that @@ -91,7 +178,7 @@ def diagnostic_properties(self): } return return_dict - def __init__(self, diagnostics): + def __init__(self, diagnostics, **kwargs): """ Args ---- @@ -100,9 +187,21 @@ def __init__(self, diagnostics): state quantities and values are the value of those quantities. The values in the dictionary will be returned when this Diagnostic is called. + input_scale_factors : dict, optional + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + diagnostic_scale_factors : dict, optional + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + update_interval : timedelta, optional + If given, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. """ self.__diagnostics = diagnostics.copy() - super(ConstantDiagnostic, self).__init__() + super(ConstantDiagnostic, self).__init__(**kwargs) def array_call(self, state): return_state = {} @@ -119,6 +218,44 @@ class RelaxationPrognostic(Prognostic): :math:`\frac{dx}{dt} = - \frac{x - x_{eq}}{\tau}` where :math:`x` is the quantity being relaxed, :math:`x_{eq}` is the equilibrium value, and :math:`\tau` is the timescale of the relaxation. + + Attributes + ---------- + input_properties : dict + A dictionary whose keys are quantities required in the state when the + object is called, and values are dictionaries which indicate 'dims' and + 'units'. + tendency_properties : dict + A dictionary whose keys are quantities for which tendencies are returned when the + object is called, and values are dictionaries which indicate 'dims' and + 'units'. + diagnostic_properties : dict + A dictionary whose keys are diagnostic quantities returned when the + object is called, and values are dictionaries which indicate 'dims' and + 'units'. + input_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which input values are scaled before being used + by this object. + tendency_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which tendency values are scaled before being + returned by this object. + diagnostic_scale_factors : dict + A (possibly empty) dictionary whose keys are quantity names and + values are floats by which diagnostic values are scaled before being + returned by this object. + update_interval : timedelta + If not None, the component will only give new output if at least + a period of update_interval has passed since the last time new + output was given. Otherwise, it would return that cached output. + tendencies_in_diagnostics : boo + A boolean indicating whether this object will put tendencies of + quantities in its diagnostic output based on first order time + differencing of output values. + name : string + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". """ @property @@ -250,6 +387,16 @@ def diagnostic_properties(self): return self._implicit.diagnostic_propertes def __init__(self, implicit, **kwargs): + """ + Initializes the TimeDifferencingWrapper. Some kwargs of Implicit + objects are not implemented, and should be applied instead on the + Implicit object which is wrapped by this one. + + Parameters + ---------- + implicit: Implicit + An Implicit component to wrap. + """ if len(kwargs) > 0: raise TypeError('Received unexpected keyword argument {}'.format( kwargs.popitem()[0])) From 05f6eb3777a93c779d4eebce0f8fdfbd65e07122 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 28 Mar 2018 17:24:05 -0700 Subject: [PATCH 40/98] Removed print statements --- sympl/_core/state.py | 1 - sympl/_core/timestepper.py | 1 - 2 files changed, 2 deletions(-) diff --git a/sympl/_core/state.py b/sympl/_core/state.py index 85d1079..b064fce 100644 --- a/sympl/_core/state.py +++ b/sympl/_core/state.py @@ -189,7 +189,6 @@ def restore_data_arrays_with_properties( out_dims.append(dims[i]) out_array = np.reshape(raw_arrays[raw_name], target_shape) else: - print(raw_arrays) if len(dims) != len(raw_arrays[raw_name].shape): raise InvalidPropertyDictError( 'Returned array for {} has shape {} ' diff --git a/sympl/_core/timestepper.py b/sympl/_core/timestepper.py index 55448ea..ac8ddbc 100644 --- a/sympl/_core/timestepper.py +++ b/sympl/_core/timestepper.py @@ -188,7 +188,6 @@ def _insert_tendencies_to_diagnostics( 'TimeStepper ({}). You must disable ' 'tendencies_in_diagnostics for this TimeStepper.'.format( tendency_name)) - print(name, input_properties, output_properties) base_units = input_properties[name]['units'] diagnostics[tendency_name] = ( (new_state[name].to_units(base_units) - state[name].to_units(base_units)) / From 6609b459aae63817ff6548a05d39b913119df382 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 29 Mar 2018 15:55:56 -0700 Subject: [PATCH 41/98] Fixed bug regarding aliasing in NetCDFMonitor --- HISTORY.rst | 1 + sympl/_components/netcdf.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index f8f368f..f15d654 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -33,6 +33,7 @@ Latest Prognostic objects, leading to unexpected value changes. * Fixed a bug where constants were missing from the string returned by get_constants_string, particularly any new constants (issue #27) +* Fixed a bug in NetCDFMonitor which led to some aliases being skipped. Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/_components/netcdf.py b/sympl/_components/netcdf.py index 0ae2829..d5af713 100644 --- a/sympl/_components/netcdf.py +++ b/sympl/_components/netcdf.py @@ -102,7 +102,7 @@ def store(self, state): # replace cached variable names with their aliases for longname, shortname in self._aliases.items(): - for full_var_name in cache_state.keys(): + for full_var_name in tuple(cache_state.keys()): # replace any string in the full variable name that matches longname # example: if longname is "temperature", shortname is "T", and # full_var_name is "temperature_tendency_from_radiation", the From b57175eb073674d44f499fda2949aed9305f94e0 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 29 Mar 2018 15:56:18 -0700 Subject: [PATCH 42/98] Began re-adding wrappers --- sympl/_core/wrappers.py | 378 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 378 insertions(+) create mode 100644 sympl/_core/wrappers.py diff --git a/sympl/_core/wrappers.py b/sympl/_core/wrappers.py new file mode 100644 index 0000000..c5f3f3f --- /dev/null +++ b/sympl/_core/wrappers.py @@ -0,0 +1,378 @@ +from .._core.base_components import ( + Prognostic, Diagnostic, ImplicitPrognostic, Implicit +) + + +class InputScalingMixin(object): + + @property + def input_properties(self): + return self.wrapped_component.input_properties + + def __init__(self, input_scale_factors=None): + self.input_scale_factors = dict() + if input_scale_factors is not None: + for input_field, value in input_scale_factors.items(): + if input_field not in self.wrapped_component.inputs: + raise ValueError( + "{} is not an input of the wrapped component.".format(input_field)) + self.input_scale_factors[input_field] = value + super(InputScalingMixin, self).__init__() + + def apply_input_scaling(self, input_state): + scaled_state = {} + for name in scaled_state.keys(): + if name in self.input_scale_factors.keys(): + scale_factor = self.input_scale_factors[name] + scaled_state[name] = input_state[name]*scale_factor + scaled_state[name].attrs.update(input_state[name].attrs) + else: + scaled_state[name] = input_state[name] + return scaled_state + + +class OutputScalingMixin(object): + + @property + def output_properties(self): + return self.wrapped_component.output_properties + + def __init__(self, output_scale_factors=None): + self.output_scale_factors = dict() + if output_scale_factors is not None: + for input_field, value in output_scale_factors.items(): + if input_field not in self.wrapped_component.inputs: + raise ValueError( + "{} is not an input of the wrapped component.".format( + input_field)) + self.output_scale_factors[input_field] = value + super(OutputScalingMixin, self).__init__() + + def apply_output_scaling(self, output_state): + scaled_outputs = {} + for name in scaled_outputs.keys(): + if name in self.output_scale_factors.keys(): + scale_factor = self.output_scale_factors[name] + scaled_outputs[name] = output_state[name] * scale_factor + scaled_outputs[name].attrs.update(output_state[name].attrs) + else: + scaled_outputs[name] = output_state[name] + return scaled_outputs + + +class DiagnosticScalingMixin(object): + + @property + def diagnostic_properties(self): + return self.wrapped_component.diagnostic_properties + + def __init__(self, diagnostic_scale_factors=None): + self.diagnostic_scale_factors = dict() + if diagnostic_scale_factors is not None: + for input_field, value in diagnostic_scale_factors.items(): + if input_field not in self.wrapped_component.inputs: + raise ValueError( + "{} is not an input of the wrapped component.".format( + input_field)) + self.diagnostic_scale_factors[input_field] = value + super(DiagnosticScalingMixin, self).__init__() + + def apply_diagnostic_scaling(self, diagnostics): + scaled_diagnostics = {} + for name in scaled_diagnostics.keys(): + if name in self.diagnostic_scale_factors.keys(): + scale_factor = self.diagnostic_scale_factors[name] + scaled_diagnostics[name] = diagnostics[name] * scale_factor + scaled_diagnostics[name].attrs.update(diagnostics[name].attrs) + else: + scaled_diagnostics[name] = diagnostics[name] + return scaled_diagnostics + + +class TendencyScalingMixin(object): + + @property + def tendency_properties(self): + return self.wrapped_component.tendency_properties + + def __init__(self, tendency_scale_factors=None): + self.tendency_scale_factors = dict() + if tendency_scale_factors is not None: + for input_field, value in tendency_scale_factors.items(): + if input_field not in self.wrapped_component.inputs: + raise ValueError( + "{} is not an input of the wrapped component.".format( + input_field)) + self.tendency_scale_factors[input_field] = value + super(TendencyScalingMixin, self).__init__() + + def apply_tendency_scaling(self, tendencies): + scaled_tendencies = {} + for name in scaled_tendencies.keys(): + if name in self.tendency_scale_factors.keys(): + scale_factor = self.tendency_scale_factors[name] + scaled_tendencies[name] = tendencies[name] * scale_factor + scaled_tendencies[name].attrs.update(tendencies[name].attrs) + else: + scaled_tendencies[name] = tendencies[name] + return scaled_tendencies + + +class ScalingWrapper(object): + + def __init__(self, component): + """ + Initializes the scaling wrapper. + + Parameters + ---------- + component + The component to be wrapped. + """ + self.wrapped_component = component + super(ScalingWrapper, self).__init__() + + +class PrognosticScalingWrapper( + ScalingWrapper, Prognostic, InputScalingMixin, DiagnosticScalingMixin, TendencyScalingMixin): + + def __call__(self, state): + input = self.apply_input_scaling(state) + tendencies, diagnostics = self.wrapped_component(input) + tendencies = self.apply_tendency_scaling(tendencies) + diagnostics = self.apply_diagnostic_scaling(diagnostics) + return tendencies, diagnostics + + +class ImplicitPrognosticScalingWrapper( + ScalingWrapper, ImplicitPrognostic, InputScalingMixin, DiagnosticScalingMixin, TendencyScalingMixin): + + def __call__(self, state, timestep): + input = self.apply_input_scaling(state) + tendencies, diagnostics = self.wrapped_component(input, timestep) + tendencies = self.apply_tendency_scaling(tendencies) + diagnostics = self.apply_diagnostic_scaling(diagnostics) + return tendencies, diagnostics + + +class DiagnosticScalingWrapper( + ScalingWrapper, Diagnostic, InputScalingMixin, DiagnosticScalingMixin): + + def __call__(self, state): + input = self.apply_input_scaling(state) + tendencies, diagnostics = self.wrapped_component(input) + tendencies = self.apply_tendency_scaling(tendencies) + diagnostics = self.apply_diagnostic_scaling(diagnostics) + return tendencies, diagnostics + + +class ImplicitScalingWrapper( + ScalingWrapper, Diagnostic, InputScalingMixin, DiagnosticScalingMixin, OutputScalingMixin): + + def __call__(self, state, timestep): + input = self.apply_input_scaling(state) + diagnostics, output = self.wrapped_component(input, timestep) + diagnostics = self.apply_diagnostic_scaling(diagnostics) + output = self.apply_output_scaling(output) + return diagnostics, output + + +class ScalingWrapper(object): + """ + Wraps any component and scales either inputs, outputs or tendencies + by a floating point value. + Example + ------- + This is how the ScaledInputOutputWrapper can be used to wrap a Prognostic. + >>> scaled_component = ScaledInputOutputWrapper( + >>> RRTMRadiation(), + >>> input_scale_factors = { + >>> 'specific_humidity' = 0.2}, + >>> tendency_scale_factors = { + >>> 'air_temperature' = 1.5}) + """ + + def __init__(self, + component, + input_scale_factors=None, + output_scale_factors=None, + tendency_scale_factors=None, + diagnostic_scale_factors=None): + """ + Initializes the ScaledInputOutputWrapper object. + Args + ---- + component : Prognostic, Implicit + The component to be wrapped. + input_scale_factors : dict + a dictionary whose keys are the inputs that will be scaled + and values are floating point scaling factors. + output_scale_factors : dict + a dictionary whose keys are the outputs that will be scaled + and values are floating point scaling factors. + tendency_scale_factors : dict + a dictionary whose keys are the tendencies that will be scaled + and values are floating point scaling factors. + diagnostic_scale_factors : dict + a dictionary whose keys are the diagnostics that will be scaled + and values are floating point scaling factors. + Returns + ------- + scaled_component : ScaledInputOutputWrapper + the scaled version of the component + Raises + ------ + TypeError + The component is not of type Implicit or Prognostic. + ValueError + The keys in the scale factors do not correspond to valid + input/output/tendency for this component. + """ + + self._input_scale_factors = dict() + if input_scale_factors is not None: + + for input_field in input_scale_factors.keys(): + if input_field not in component.inputs: + raise ValueError( + "{} is not a valid input quantity.".format(input_field)) + + self._input_scale_factors = input_scale_factors + + self._diagnostic_scale_factors = dict() + if diagnostic_scale_factors is not None: + + for diagnostic_field in diagnostic_scale_factors.keys(): + if diagnostic_field not in component.diagnostics: + raise ValueError( + "{} is not a valid diagnostic quantity.".format(diagnostic_field)) + + self._diagnostic_scale_factors = diagnostic_scale_factors + + if hasattr(component, 'input_properties') and hasattr(component, 'output_properties'): + + self._output_scale_factors = dict() + if output_scale_factors is not None: + + for output_field in output_scale_factors.keys(): + if output_field not in component.outputs: + raise ValueError( + "{} is not a valid output quantity.".format(output_field)) + + self._output_scale_factors = output_scale_factors + self._component_type = 'Implicit' + + elif hasattr(component, 'input_properties') and hasattr(component, 'tendency_properties'): + + self._tendency_scale_factors = dict() + if tendency_scale_factors is not None: + + for tendency_field in tendency_scale_factors.keys(): + if tendency_field not in component.tendencies: + raise ValueError( + "{} is not a valid tendency quantity.".format(tendency_field)) + + self._tendency_scale_factors = tendency_scale_factors + self._component_type = 'Prognostic' + + elif hasattr(component, 'input_properties') and hasattr(component, 'diagnostic_properties'): + self._component_type = 'Diagnostic' + else: + raise TypeError( + "Component must be either of type Implicit or Prognostic or Diagnostic") + + self._component = component + + def __getattr__(self, item): + return getattr(self._component, item) + + def __call__(self, state, timestep=None): + + scaled_state = {} + if 'time' in state: + scaled_state['time'] = state['time'] + + for input_field in self.inputs: + if input_field in self._input_scale_factors: + scale_factor = self._input_scale_factors[input_field] + scaled_state[input_field] = state[input_field]*float(scale_factor) + else: + scaled_state[input_field] = state[input_field] + + if self._component_type == 'Implicit': + diagnostics, new_state = self._component(scaled_state, timestep) + + for output_field in self._output_scale_factors.keys(): + scale_factor = self._output_scale_factors[output_field] + new_state[output_field] *= float(scale_factor) + + for diagnostic_field in self._diagnostic_scale_factors.keys(): + scale_factor = self._diagnostic_scale_factors[diagnostic_field] + diagnostics[diagnostic_field] *= float(scale_factor) + + return diagnostics, new_state + elif self._component_type == 'Prognostic': + tendencies, diagnostics = self._component(scaled_state) + + for tend_field in self._tendency_scale_factors.keys(): + scale_factor = self._tendency_scale_factors[tend_field] + tendencies[tend_field] *= float(scale_factor) + + for diagnostic_field in self._diagnostic_scale_factors.keys(): + scale_factor = self._diagnostic_scale_factors[diagnostic_field] + diagnostics[diagnostic_field] *= float(scale_factor) + + return tendencies, diagnostics + elif self._component_type == 'Diagnostic': + diagnostics = self._component(scaled_state) + + for diagnostic_field in self._diagnostic_scale_factors.keys(): + scale_factor = self._diagnostic_scale_factors[diagnostic_field] + diagnostics[diagnostic_field] *= float(scale_factor) + + return diagnostics + else: # Should never reach this + raise ValueError( + 'Unknown component type, seems to be a bug in ScalingWrapper') + + +class UpdateFrequencyWrapper(object): + """ + Wraps a prognostic object so that when it is called, it only computes new + output if sufficient time has passed, and otherwise returns its last + computed output. The Delayed object requires that the 'time' attribute is + set in the state, in addition to any requirements of the Prognostic + Example + ------- + This how the wrapper should be used on a fictional Prognostic class + called MyPrognostic. + >>> from datetime import timedelta + >>> prognostic = UpdateFrequencyWrapper(MyPrognostic(), timedelta(hours=1)) + """ + + def __init__(self, prognostic, update_timedelta): + """ + Initialize the UpdateFrequencyWrapper object. + Args + ---- + prognostic : Prognostic + The object to be wrapped. + update_timedelta : timedelta + The amount that state['time'] must differ from when output + was cached before new output is computed. + """ + self._prognostic = prognostic + self._update_timedelta = update_timedelta + self._cached_output = None + self._last_update_time = None + + def __call__(self, state, **kwargs): + if ((self._last_update_time is None) or + (state['time'] >= self._last_update_time + + self._update_timedelta)): + self._cached_output = self._prognostic(state, **kwargs) + self._last_update_time = state['time'] + return self._cached_output + + def __getattr__(self, item): + return getattr(self._prognostic, item) From d91ea635e19459be8bbed21bca04fedbc4b6a179 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 29 Mar 2018 16:11:15 -0700 Subject: [PATCH 43/98] Corrected init of PrognosticScalingWrapper, others to follow --- sympl/_core/wrappers.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/sympl/_core/wrappers.py b/sympl/_core/wrappers.py index c5f3f3f..ab7bc8a 100644 --- a/sympl/_core/wrappers.py +++ b/sympl/_core/wrappers.py @@ -118,9 +118,11 @@ def apply_tendency_scaling(self, tendencies): return scaled_tendencies -class ScalingWrapper(object): +class PrognosticScalingWrapper( + Prognostic, InputScalingMixin, DiagnosticScalingMixin, TendencyScalingMixin): - def __init__(self, component): + def __init__(self, component, input_scale_factors, diagnostic_scale_factors, + tendency_scale_factors): """ Initializes the scaling wrapper. @@ -128,13 +130,15 @@ def __init__(self, component): ---------- component The component to be wrapped. + input_scale_factors : dict + diagnostic_scale_factors : dict + tendency_scale_factors : dict """ self.wrapped_component = component - super(ScalingWrapper, self).__init__() - - -class PrognosticScalingWrapper( - ScalingWrapper, Prognostic, InputScalingMixin, DiagnosticScalingMixin, TendencyScalingMixin): + super(ScalingWrapper, self).__init__( + input_scale_factors=input_scale_factors, + diagnostic_scale_factors=diagnostic_scale_factors, + tendency_scale_factors=tendency_scale_factors) def __call__(self, state): input = self.apply_input_scaling(state) @@ -145,7 +149,7 @@ def __call__(self, state): class ImplicitPrognosticScalingWrapper( - ScalingWrapper, ImplicitPrognostic, InputScalingMixin, DiagnosticScalingMixin, TendencyScalingMixin): + ImplicitPrognostic, InputScalingMixin, DiagnosticScalingMixin, TendencyScalingMixin): def __call__(self, state, timestep): input = self.apply_input_scaling(state) @@ -156,7 +160,7 @@ def __call__(self, state, timestep): class DiagnosticScalingWrapper( - ScalingWrapper, Diagnostic, InputScalingMixin, DiagnosticScalingMixin): + Diagnostic, InputScalingMixin, DiagnosticScalingMixin): def __call__(self, state): input = self.apply_input_scaling(state) @@ -167,7 +171,7 @@ def __call__(self, state): class ImplicitScalingWrapper( - ScalingWrapper, Diagnostic, InputScalingMixin, DiagnosticScalingMixin, OutputScalingMixin): + Diagnostic, InputScalingMixin, DiagnosticScalingMixin, OutputScalingMixin): def __call__(self, state, timestep): input = self.apply_input_scaling(state) From 4f7a73016c2e3b1cee055ecb0db9343e0b593102 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Tue, 3 Apr 2018 15:59:12 -0700 Subject: [PATCH 44/98] Restored UpdateFrequencyWrapper and ScalingWrapper, changed isinstance isinstance checks on component base types now check that the instance satisfies properties of that base type rather than strict type checking. This allows a single wrapper type to have instances that can be any of a number of component types, depending on the wrapped component type. UpdateFrequencyWrapper is again a wrapper, which can now wrap any component type. Similarly for ScalingWrapper. Documentation still has to be updated for these wrappers. --- HISTORY.rst | 5 + sympl/__init__.py | 2 + sympl/_core/base_components.py | 468 +++++------- sympl/_core/state.py | 9 +- sympl/_core/wrappers.py | 352 +++------ tests/test_base_components.py | 1227 ++++++++++++++++++-------------- tests/test_wrapper.py | 678 ++++++++++++++++++ 7 files changed, 1667 insertions(+), 1074 deletions(-) create mode 100644 tests/test_wrapper.py diff --git a/HISTORY.rst b/HISTORY.rst index f15d654..8d5b0ca 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -34,6 +34,11 @@ Latest * Fixed a bug where constants were missing from the string returned by get_constants_string, particularly any new constants (issue #27) * Fixed a bug in NetCDFMonitor which led to some aliases being skipped. +* Modified class checking on components so that components which satisfy the + component's API will be recognized as instances using isinstance(obj, Class). + Right now this only checks for the presence and lack of presence of + component attributes, and correct signature of __call__. Later it may also + check properties dictionaries for consistency, or perform other checks. Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/__init__.py b/sympl/__init__.py index 500c1f8..5a7ebfe 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -26,6 +26,7 @@ PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, TimeDifferencingWrapper) +from ._core.wrappers import UpdateFrequencyWrapper, ScalingWrapper from ._core.time import datetime, timedelta __version__ = '0.3.2' @@ -46,5 +47,6 @@ get_component_aliases, PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, + UpdateFrequencyWrapper, ScalingWrapper, datetime, timedelta ) diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index b8c73f9..da2c344 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -4,6 +4,14 @@ from .exceptions import ( InvalidPropertyDictError, ComponentExtraOutputError, ComponentMissingOutputError, InvalidStateError) +from inspect import getargspec + + +def option_or_default(option, default): + if option is None: + return default + else: + return option def apply_scale_factors(array_state, scale_factors): @@ -11,6 +19,77 @@ def apply_scale_factors(array_state, scale_factors): array_state[key] *= factor +def is_component_class(cls): + return any(issubclass(cls, cls2) for cls2 in (Implicit, Prognostic, ImplicitPrognostic, Diagnostic)) + + +def is_component_base_class(cls): + return cls in (Implicit, Prognostic, ImplicitPrognostic, Diagnostic) + + +class ComponentMeta(abc.ABCMeta): + + def __instancecheck__(self, instance): + if is_component_class(instance.__class__) or not is_component_base_class(self): + return issubclass(instance.__class__, self) + else: + check_attributes = ( + 'input_properties', + 'tendency_properties', + 'diagnostic_properties', + 'output_properties', + '__call__', + 'array_call', + 'tendencies_in_diagnostics', + 'name', + ) + required_attributes = list( + att for att in check_attributes if hasattr(self, att) + ) + disallowed_attributes = list( + att for att in check_attributes if att not in required_attributes + ) + if 'name' in disallowed_attributes: # name is always allowed + disallowed_attributes.remove('name') + has_attributes = ( + all(hasattr(instance, att) for att in required_attributes) and + not any(hasattr(instance, att) for att in disallowed_attributes) + ) + if hasattr(self, '__call__') and not hasattr(instance, '__call__'): + return False + elif hasattr(self, '__call__'): + timestep_in_class_call = 'timestep' in getargspec(self.__call__).args + instance_argspec = getargspec(instance.__call__) + timestep_in_instance_call = 'timestep' in instance_argspec.args + instance_defaults = {} + if instance_argspec.defaults is not None: + n = len(instance_argspec.args) - 1 + for i, default in enumerate(reversed(instance_argspec.defaults)): + instance_defaults[instance_argspec.args[n-i]] = default + timestep_optional = ( + 'timestep' in instance_defaults.keys() and instance_defaults['timestep'] is None) + has_correct_spec = (timestep_in_class_call == timestep_in_instance_call) or timestep_optional + else: + raise RuntimeError( + 'Cannot check instance type on component subclass that has ' + 'no __call__ method') + return has_attributes and has_correct_spec + + +def check_overlapping_aliases(properties, properties_name): + defined_aliases = set() + for name, properties in properties.items(): + if 'alias' in properties.keys(): + if properties['alias'] not in defined_aliases: + defined_aliases.add(properties['alias']) + else: + raise InvalidPropertyDictError( + 'Multiple quantities map to alias {} in {} ' + 'properties'.format( + properties['alias'], properties_name) + ) + + class InputMixin(object): def __init__(self): @@ -22,6 +101,7 @@ def __init__(self): raise InvalidPropertyDictError( 'Input properties do not have dims defined for {}'.format(name) ) + check_overlapping_aliases(self.input_properties, 'input') super(InputMixin, self).__init__() def _check_inputs(self, state): @@ -41,6 +121,7 @@ def __init__(self): raise InvalidPropertyDictError( 'Tendency properties do not have dims defined for {}'.format(name) ) + check_overlapping_aliases(self.tendency_properties, 'tendency') super(TendencyMixin, self).__init__() @property @@ -94,6 +175,7 @@ def __init__(self): raise InvalidPropertyDictError( 'Diagnostic properties do not have dims defined for {}'.format(name) ) + check_overlapping_aliases(self.diagnostic_properties, 'diagnostic') super(DiagnosticMixin, self).__init__() @property @@ -147,6 +229,7 @@ def __init__(self): raise InvalidPropertyDictError( 'Output properties do not have dims defined for {}'.format(name) ) + check_overlapping_aliases(self.output_properties, 'output') super(OutputMixin, self).__init__() @property @@ -209,34 +292,21 @@ class Implicit(DiagnosticMixin, OutputMixin, InputMixin): for the new state are returned when the object is called, and values are dictionaries which indicate 'dims' and 'units'. - input_scale_factors : dict - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which input values are scaled before being used - by this object. - output_scale_factors : dict - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which output values are scaled before being - returned by this object. - diagnostic_scale_factors : dict - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which diagnostic values are scaled before being - returned by this object. tendencies_in_diagnostics : bool A boolean indicating whether this object will put tendencies of quantities in its diagnostic output based on first order time differencing of output values. - update_interval : timedelta - If not None, the component will only give new output if at least - a period of update_interval has passed since the last time new - output was given. Otherwise, it would return that cached output. time_unit_name : str The unit to use for time differencing when putting tendencies in diagnostics. time_unit_timedelta: timedelta A timedelta corresponding to a single time unit as used for time differencing when putting tendencies in diagnostics. + name : string + A label to be used for this object, for example as would be used for + Y in the name "X_tendency_from_Y". """ - __metaclass__ = abc.ABCMeta + __metaclass__ = ComponentMeta time_unit_name = 's' time_unit_timedelta = timedelta(seconds=1) @@ -277,59 +347,23 @@ def __repr__(self): self._making_repr = False return return_value - def __init__( - self, input_scale_factors=None, output_scale_factors=None, - diagnostic_scale_factors=None, tendencies_in_diagnostics=False, - update_interval=None, name=None): + def __init__(self, tendencies_in_diagnostics=False, name=None): """ Initializes the Implicit object. Args ---- - input_scale_factors : dict, optional - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which input values are scaled before being used - by this object. - output_scale_factors : dict, optional - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which output values are scaled before being - returned by this object. - diagnostic_scale_factors : dict, optional - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which diagnostic values are scaled before being - returned by this object. tendencies_in_diagnostics : bool, optional A boolean indicating whether this object will put tendencies of quantities in its diagnostic output based on first order time differencing of output values. - update_interval : timedelta, optional - If given, the component will only give new output if at least - a period of update_interval has passed since the last time new - output was given. Otherwise, it would return that cached output. name : string, optional A label to be used for this object, for example as would be used for Y in the name "X_tendency_from_Y". By default the class name in lowercase is used. """ - if input_scale_factors is not None: - self.input_scale_factors = input_scale_factors - else: - self.input_scale_factors = {} - if output_scale_factors is not None: - self.output_scale_factors = output_scale_factors - else: - self.output_scale_factors = {} - if diagnostic_scale_factors is not None: - self.diagnostic_scale_factors = diagnostic_scale_factors - else: - self.diagnostic_scale_factors = {} self._tendencies_in_diagnostics = tendencies_in_diagnostics - self.update_interval = update_interval - self._last_update_time = None - if name is None: - self.name = self.__class__.__name__.lower() - else: - self.name = name + self.name = name or self.__class__.__name__.lower() if tendencies_in_diagnostics: self._added_tendency_properties = self._insert_tendency_properties() else: @@ -426,29 +460,22 @@ def __call__(self, state, timestep): If state is not a valid input for the Implicit instance for other reasons. """ - if (self.update_interval is None or - self._last_update_time is None or - state['time'] >= self._last_update_time + self.update_interval): - self._check_inputs(state) - raw_state = get_numpy_arrays_with_properties(state, self.input_properties) - raw_state['time'] = state['time'] - apply_scale_factors(raw_state, self.input_scale_factors) - raw_diagnostics, raw_new_state = self.array_call(raw_state, timestep) - self._check_diagnostics(raw_diagnostics) - self._check_outputs(raw_new_state) - apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) - apply_scale_factors(raw_new_state, self.output_scale_factors) - if self.tendencies_in_diagnostics: - self._insert_tendencies_to_diagnostics( - raw_state, raw_new_state, timestep, raw_diagnostics) - self._diagnostics = restore_data_arrays_with_properties( - raw_diagnostics, self.diagnostic_properties, - state, self.input_properties) - self._new_state = restore_data_arrays_with_properties( - raw_new_state, self.output_properties, - state, self.input_properties) - self._last_update_time = state['time'] - return self._diagnostics, self._new_state + self._check_inputs(state) + raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + raw_state['time'] = state['time'] + raw_diagnostics, raw_new_state = self.array_call(raw_state, timestep) + self._check_diagnostics(raw_diagnostics) + self._check_outputs(raw_new_state) + if self.tendencies_in_diagnostics: + self._insert_tendencies_to_diagnostics( + raw_state, raw_new_state, timestep, raw_diagnostics) + diagnostics = restore_data_arrays_with_properties( + raw_diagnostics, self.diagnostic_properties, + state, self.input_properties) + new_state = restore_data_arrays_with_properties( + raw_new_state, self.output_properties, + state, self.input_properties) + return diagnostics, new_state def _insert_tendencies_to_diagnostics( self, raw_state, raw_new_state, timestep, raw_diagnostics): @@ -501,23 +528,7 @@ class Prognostic(DiagnosticMixin, TendencyMixin, InputMixin): A dictionary whose keys are diagnostic quantities returned when the object is called, and values are dictionaries which indicate 'dims' and 'units'. - input_scale_factors : dict - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which input values are scaled before being used - by this object. - tendency_scale_factors : dict - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which tendency values are scaled before being - returned by this object. - diagnostic_scale_factors : dict - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which diagnostic values are scaled before being - returned by this object. - update_interval : timedelta - If not None, the component will only give new output if at least - a period of update_interval has passed since the last time new - output was given. Otherwise, it would return that cached output. - tendencies_in_diagnostics : boo + tendencies_in_diagnostics : bool A boolean indicating whether this object will put tendencies of quantities in its diagnostic output based on first order time differencing of output values. @@ -525,7 +536,7 @@ class Prognostic(DiagnosticMixin, TendencyMixin, InputMixin): A label to be used for this object, for example as would be used for Y in the name "X_tendency_from_Y". """ - __metaclass__ = abc.ABCMeta + __metaclass__ = ComponentMeta @abc.abstractproperty def input_properties(self): @@ -539,6 +550,8 @@ def tendency_properties(self): def diagnostic_properties(self): return {} + name = None + def __str__(self): return ( 'instance of {}(Prognostic)\n' @@ -563,31 +576,12 @@ def __repr__(self): self._making_repr = False return return_value - def __init__( - self, input_scale_factors=None, tendency_scale_factors=None, - diagnostic_scale_factors=None, update_interval=None, - tendencies_in_diagnostics=False, name=None): + def __init__(self, tendencies_in_diagnostics=False, name=None): """ Initializes the Implicit object. Args ---- - input_scale_factors : dict, optional - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which input values are scaled before being used - by this object. - tendency_scale_factors : dict, optional - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which tendency values are scaled before being - returned by this object. - diagnostic_scale_factors : dict, optional - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which diagnostic values are scaled before being - returned by this object. - update_interval : timedelta, optional - If given, the component will only give new output if at least - a period of update_interval has passed since the last time new - output was given. Otherwise, it would return that cached output. tendencies_in_diagnostics : bool, optional A boolean indicating whether this object will put tendencies of quantities in its diagnostic output. @@ -596,25 +590,8 @@ def __init__( Y in the name "X_tendency_from_Y". By default the class name in lowercase is used. """ - if input_scale_factors is not None: - self.input_scale_factors = input_scale_factors - else: - self.input_scale_factors = {} - if tendency_scale_factors is not None: - self.tendency_scale_factors = tendency_scale_factors - else: - self.tendency_scale_factors = {} - if diagnostic_scale_factors is not None: - self.diagnostic_scale_factors = diagnostic_scale_factors - else: - self.diagnostic_scale_factors = {} - self.update_interval = update_interval - self._last_update_time = None self._tendencies_in_diagnostics = tendencies_in_diagnostics - if name is None: - self.name = self.__class__.__name__ - else: - self.name = name + self.name = name or self.__class__.__name__ self._added_diagnostic_names = [] if self.tendencies_in_diagnostics: self._insert_tendency_properties() @@ -668,33 +645,26 @@ def __call__(self, state): InvalidStateError If state is not a valid input for the Prognostic instance. """ - if (self.update_interval is None or - self._last_update_time is None or - state['time'] >= self._last_update_time + self.update_interval): - raw_state = get_numpy_arrays_with_properties(state, self.input_properties) - raw_state['time'] = state['time'] - apply_scale_factors(raw_state, self.input_scale_factors) - raw_tendencies, raw_diagnostics = self.array_call(raw_state) - self._check_tendencies(raw_tendencies) - self._check_diagnostics(raw_diagnostics) - apply_scale_factors(raw_tendencies, self.tendency_scale_factors) - apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) - self._tendencies = restore_data_arrays_with_properties( - raw_tendencies, self.tendency_properties, - state, self.input_properties) - self._diagnostics = restore_data_arrays_with_properties( - raw_diagnostics, self.diagnostic_properties, - state, self.input_properties, - ignore_names=self._added_diagnostic_names) - if self.tendencies_in_diagnostics: - self._insert_tendencies_to_diagnostics() - self._last_update_time = state['time'] - return self._tendencies, self._diagnostics - - def _insert_tendencies_to_diagnostics(self): - for name, value in self._tendencies.items(): + raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + raw_state['time'] = state['time'] + raw_tendencies, raw_diagnostics = self.array_call(raw_state) + self._check_tendencies(raw_tendencies) + self._check_diagnostics(raw_diagnostics) + tendencies = restore_data_arrays_with_properties( + raw_tendencies, self.tendency_properties, + state, self.input_properties) + diagnostics = restore_data_arrays_with_properties( + raw_diagnostics, self.diagnostic_properties, + state, self.input_properties, + ignore_names=self._added_diagnostic_names) + if self.tendencies_in_diagnostics: + self._insert_tendencies_to_diagnostics(tendencies, diagnostics) + return tendencies, diagnostics + + def _insert_tendencies_to_diagnostics(self, tendencies, diagnostics): + for name, value in tendencies.items(): tendency_name = self._get_tendency_name(name) - self._diagnostics[tendency_name] = value + diagnostics[tendency_name] = value def _check_missing_diagnostics(self, diagnostics_dict): missing_diagnostics = set() @@ -752,27 +722,7 @@ class ImplicitPrognostic(DiagnosticMixin, TendencyMixin, InputMixin): A dictionary whose keys are diagnostic quantities returned when the object is called, and values are dictionaries which indicate 'dims' and 'units'. - input_scale_factors : dict - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which input values are scaled before being used - by this object. - tendency_scale_factors : dict - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which tendency values are scaled before being - returned by this object. - diagnostic_scale_factors : dict - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which diagnostic values are scaled before being - returned by this object. - update_interval : timedelta - If not None, the component will only give new output if at least - a period of update_interval has passed since the last time new - output was given. Otherwise, it would return that cached output. - name : string - A label to be used for this object, for example as would be used for - Y in the name "X_tendency_from_Y". By default the class name in - lowercase is used. - tendencies_in_diagnostics : boo + tendencies_in_diagnostics : bool A boolean indicating whether this object will put tendencies of quantities in its diagnostic output based on first order time differencing of output values. @@ -780,7 +730,7 @@ class ImplicitPrognostic(DiagnosticMixin, TendencyMixin, InputMixin): A label to be used for this object, for example as would be used for Y in the name "X_tendency_from_Y". """ - __metaclass__ = abc.ABCMeta + __metaclass__ = ComponentMeta @abc.abstractproperty def input_properties(self): @@ -794,6 +744,8 @@ def tendency_properties(self): def diagnostic_properties(self): return {} + name = None + def __str__(self): return ( 'instance of {}(Prognostic)\n' @@ -816,31 +768,12 @@ def __repr__(self): self._making_repr = False return return_value - def __init__( - self, input_scale_factors=None, tendency_scale_factors=None, - diagnostic_scale_factors=None, update_interval=None, - tendencies_in_diagnostics=False, name=None): + def __init__(self, tendencies_in_diagnostics=False, name=None): """ Initializes the Implicit object. Args ---- - input_scale_factors : dict, optional - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which input values are scaled before being used - by this object. - tendency_scale_factors : dict, optional - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which tendency values are scaled before being - returned by this object. - diagnostic_scale_factors : dict, optional - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which diagnostic values are scaled before being - returned by this object. - update_interval : timedelta, optional - If given, the component will only give new output if at least - a period of update_interval has passed since the last time new - output was given. Otherwise, it would return that cached output. tendencies_in_diagnostics : bool, optional A boolean indicating whether this object will put tendencies of quantities in its diagnostic output. @@ -849,25 +782,8 @@ def __init__( Y in the name "X_tendency_from_Y". By default the class name in lowercase is used. """ - if input_scale_factors is not None: - self.input_scale_factors = input_scale_factors - else: - self.input_scale_factors = {} - if tendency_scale_factors is not None: - self.tendency_scale_factors = tendency_scale_factors - else: - self.tendency_scale_factors = {} - if diagnostic_scale_factors is not None: - self.diagnostic_scale_factors = diagnostic_scale_factors - else: - self.diagnostic_scale_factors = {} - self.update_interval = update_interval - self._last_update_time = None self._tendencies_in_diagnostics = tendencies_in_diagnostics - if name is None: - self.name = self.__class__.__name__ - else: - self.name = name + self.name = name or self.__class__.__name__ self._added_diagnostic_names = [] if self.tendencies_in_diagnostics: self._insert_tendency_properties() @@ -923,33 +839,27 @@ def __call__(self, state, timestep): InvalidStateError If state is not a valid input for the Prognostic instance. """ - if (self.update_interval is None or - self._last_update_time is None or - state['time'] >= self._last_update_time + self.update_interval): - raw_state = get_numpy_arrays_with_properties(state, self.input_properties) - raw_state['time'] = state['time'] - apply_scale_factors(raw_state, self.input_scale_factors) - raw_tendencies, raw_diagnostics = self.array_call(raw_state, timestep) - self._check_tendencies(raw_tendencies) - self._check_diagnostics(raw_diagnostics) - apply_scale_factors(raw_tendencies, self.tendency_scale_factors) - apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) - self._tendencies = restore_data_arrays_with_properties( - raw_tendencies, self.tendency_properties, - state, self.input_properties) - self._diagnostics = restore_data_arrays_with_properties( - raw_diagnostics, self.diagnostic_properties, - state, self.input_properties, - ignore_names=self._added_diagnostic_names) - if self.tendencies_in_diagnostics: - self._insert_tendencies_to_diagnostics() - self._last_update_time = state['time'] - return self._tendencies, self._diagnostics - - def _insert_tendencies_to_diagnostics(self): - for name, value in self._tendencies.items(): + raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + raw_state['time'] = state['time'] + raw_tendencies, raw_diagnostics = self.array_call(raw_state, timestep) + self._check_tendencies(raw_tendencies) + self._check_diagnostics(raw_diagnostics) + tendencies = restore_data_arrays_with_properties( + raw_tendencies, self.tendency_properties, + state, self.input_properties) + diagnostics = restore_data_arrays_with_properties( + raw_diagnostics, self.diagnostic_properties, + state, self.input_properties, + ignore_names=self._added_diagnostic_names) + if self.tendencies_in_diagnostics: + self._insert_tendencies_to_diagnostics(tendencies, diagnostics) + self._last_update_time = state['time'] + return tendencies, diagnostics + + def _insert_tendencies_to_diagnostics(self, tendencies, diagnostics): + for name, value in tendencies.items(): tendency_name = self._get_tendency_name(name) - self._diagnostics[tendency_name] = value + diagnostics[tendency_name] = value def _check_missing_diagnostics(self, diagnostics_dict): missing_diagnostics = set() @@ -1004,20 +914,8 @@ class Diagnostic(DiagnosticMixin, InputMixin): A dictionary whose keys are diagnostic quantities returned when the object is called, and values are dictionaries which indicate 'dims' and 'units'. - input_scale_factors : dict - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which input values are scaled before being used - by this object. - diagnostic_scale_factors : dict - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which diagnostic values are scaled before being - returned by this object. - update_interval : timedelta - If not None, the component will only give new output if at least - a period of update_interval has passed since the last time new - output was given. Otherwise, it would return that cached output. """ - __metaclass__ = abc.ABCMeta + __metaclass__ = ComponentMeta @abc.abstractproperty def input_properties(self): @@ -1048,38 +946,10 @@ def __repr__(self): self._making_repr = False return return_value - def __init__( - self, input_scale_factors=None, diagnostic_scale_factors=None, - update_interval=None): + def __init__(self): """ Initializes the Implicit object. - - Args - ---- - input_scale_factors : dict, optional - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which input values are scaled before being used - by this object. - diagnostic_scale_factors : dict, optional - A (possibly empty) dictionary whose keys are quantity names and - values are floats by which diagnostic values are scaled before being - returned by this object. - update_interval : timedelta, optional - If given, the component will only give new output if at least - a period of update_interval has passed since the last time new - output was given. Otherwise, it would return that cached output. """ - if input_scale_factors is not None: - self.input_scale_factors = input_scale_factors - else: - self.input_scale_factors = {} - if diagnostic_scale_factors is not None: - self.diagnostic_scale_factors = diagnostic_scale_factors - else: - self.diagnostic_scale_factors = {} - self.update_interval = update_interval - self._last_update_time = None - self._diagnostics = None super(Diagnostic, self).__init__() def __call__(self, state): @@ -1105,20 +975,14 @@ def __call__(self, state): InvalidStateError If state is not a valid input for the Prognostic instance. """ - if (self.update_interval is None or - self._last_update_time is None or - state['time'] >= self._last_update_time + self.update_interval): - raw_state = get_numpy_arrays_with_properties(state, self.input_properties) - raw_state['time'] = state['time'] - apply_scale_factors(raw_state, self.input_scale_factors) - raw_diagnostics = self.array_call(raw_state) - self._check_diagnostics(raw_diagnostics) - apply_scale_factors(raw_diagnostics, self.diagnostic_scale_factors) - self._diagnostics = restore_data_arrays_with_properties( - raw_diagnostics, self.diagnostic_properties, - state, self.input_properties) - self._last_update_time = state['time'] - return self._diagnostics + raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + raw_state['time'] = state['time'] + raw_diagnostics = self.array_call(raw_state) + self._check_diagnostics(raw_diagnostics) + diagnostics = restore_data_arrays_with_properties( + raw_diagnostics, self.diagnostic_properties, + state, self.input_properties) + return diagnostics @abc.abstractmethod def array_call(self, state): diff --git a/sympl/_core/state.py b/sympl/_core/state.py index b064fce..d10fab6 100644 --- a/sympl/_core/state.py +++ b/sympl/_core/state.py @@ -187,7 +187,14 @@ def restore_data_arrays_with_properties( else: target_shape.append(length) out_dims.append(dims[i]) - out_array = np.reshape(raw_arrays[raw_name], target_shape) + try: + out_array = np.reshape(raw_arrays[raw_name], target_shape) + except ValueError: + raise InvalidPropertyDictError( + 'Failed to restore shape for output {} with raw shape {} ' + 'and target shape {}, are the output dims {} correct?'.format( + name, raw_arrays[raw_name].shape, target_shape, + out_dims_property[name])) else: if len(dims) != len(raw_arrays[raw_name].shape): raise InvalidPropertyDictError( diff --git a/sympl/_core/wrappers.py b/sympl/_core/wrappers.py index ab7bc8a..6dcea27 100644 --- a/sympl/_core/wrappers.py +++ b/sympl/_core/wrappers.py @@ -3,192 +3,14 @@ ) -class InputScalingMixin(object): - - @property - def input_properties(self): - return self.wrapped_component.input_properties - - def __init__(self, input_scale_factors=None): - self.input_scale_factors = dict() - if input_scale_factors is not None: - for input_field, value in input_scale_factors.items(): - if input_field not in self.wrapped_component.inputs: - raise ValueError( - "{} is not an input of the wrapped component.".format(input_field)) - self.input_scale_factors[input_field] = value - super(InputScalingMixin, self).__init__() - - def apply_input_scaling(self, input_state): - scaled_state = {} - for name in scaled_state.keys(): - if name in self.input_scale_factors.keys(): - scale_factor = self.input_scale_factors[name] - scaled_state[name] = input_state[name]*scale_factor - scaled_state[name].attrs.update(input_state[name].attrs) - else: - scaled_state[name] = input_state[name] - return scaled_state - - -class OutputScalingMixin(object): - - @property - def output_properties(self): - return self.wrapped_component.output_properties - - def __init__(self, output_scale_factors=None): - self.output_scale_factors = dict() - if output_scale_factors is not None: - for input_field, value in output_scale_factors.items(): - if input_field not in self.wrapped_component.inputs: - raise ValueError( - "{} is not an input of the wrapped component.".format( - input_field)) - self.output_scale_factors[input_field] = value - super(OutputScalingMixin, self).__init__() - - def apply_output_scaling(self, output_state): - scaled_outputs = {} - for name in scaled_outputs.keys(): - if name in self.output_scale_factors.keys(): - scale_factor = self.output_scale_factors[name] - scaled_outputs[name] = output_state[name] * scale_factor - scaled_outputs[name].attrs.update(output_state[name].attrs) - else: - scaled_outputs[name] = output_state[name] - return scaled_outputs - - -class DiagnosticScalingMixin(object): - - @property - def diagnostic_properties(self): - return self.wrapped_component.diagnostic_properties - - def __init__(self, diagnostic_scale_factors=None): - self.diagnostic_scale_factors = dict() - if diagnostic_scale_factors is not None: - for input_field, value in diagnostic_scale_factors.items(): - if input_field not in self.wrapped_component.inputs: - raise ValueError( - "{} is not an input of the wrapped component.".format( - input_field)) - self.diagnostic_scale_factors[input_field] = value - super(DiagnosticScalingMixin, self).__init__() - - def apply_diagnostic_scaling(self, diagnostics): - scaled_diagnostics = {} - for name in scaled_diagnostics.keys(): - if name in self.diagnostic_scale_factors.keys(): - scale_factor = self.diagnostic_scale_factors[name] - scaled_diagnostics[name] = diagnostics[name] * scale_factor - scaled_diagnostics[name].attrs.update(diagnostics[name].attrs) - else: - scaled_diagnostics[name] = diagnostics[name] - return scaled_diagnostics - - -class TendencyScalingMixin(object): - - @property - def tendency_properties(self): - return self.wrapped_component.tendency_properties - - def __init__(self, tendency_scale_factors=None): - self.tendency_scale_factors = dict() - if tendency_scale_factors is not None: - for input_field, value in tendency_scale_factors.items(): - if input_field not in self.wrapped_component.inputs: - raise ValueError( - "{} is not an input of the wrapped component.".format( - input_field)) - self.tendency_scale_factors[input_field] = value - super(TendencyScalingMixin, self).__init__() - - def apply_tendency_scaling(self, tendencies): - scaled_tendencies = {} - for name in scaled_tendencies.keys(): - if name in self.tendency_scale_factors.keys(): - scale_factor = self.tendency_scale_factors[name] - scaled_tendencies[name] = tendencies[name] * scale_factor - scaled_tendencies[name].attrs.update(tendencies[name].attrs) - else: - scaled_tendencies[name] = tendencies[name] - return scaled_tendencies - - -class PrognosticScalingWrapper( - Prognostic, InputScalingMixin, DiagnosticScalingMixin, TendencyScalingMixin): - - def __init__(self, component, input_scale_factors, diagnostic_scale_factors, - tendency_scale_factors): - """ - Initializes the scaling wrapper. - - Parameters - ---------- - component - The component to be wrapped. - input_scale_factors : dict - diagnostic_scale_factors : dict - tendency_scale_factors : dict - """ - self.wrapped_component = component - super(ScalingWrapper, self).__init__( - input_scale_factors=input_scale_factors, - diagnostic_scale_factors=diagnostic_scale_factors, - tendency_scale_factors=tendency_scale_factors) - - def __call__(self, state): - input = self.apply_input_scaling(state) - tendencies, diagnostics = self.wrapped_component(input) - tendencies = self.apply_tendency_scaling(tendencies) - diagnostics = self.apply_diagnostic_scaling(diagnostics) - return tendencies, diagnostics - - -class ImplicitPrognosticScalingWrapper( - ImplicitPrognostic, InputScalingMixin, DiagnosticScalingMixin, TendencyScalingMixin): - - def __call__(self, state, timestep): - input = self.apply_input_scaling(state) - tendencies, diagnostics = self.wrapped_component(input, timestep) - tendencies = self.apply_tendency_scaling(tendencies) - diagnostics = self.apply_diagnostic_scaling(diagnostics) - return tendencies, diagnostics - - -class DiagnosticScalingWrapper( - Diagnostic, InputScalingMixin, DiagnosticScalingMixin): - - def __call__(self, state): - input = self.apply_input_scaling(state) - tendencies, diagnostics = self.wrapped_component(input) - tendencies = self.apply_tendency_scaling(tendencies) - diagnostics = self.apply_diagnostic_scaling(diagnostics) - return tendencies, diagnostics - - -class ImplicitScalingWrapper( - Diagnostic, InputScalingMixin, DiagnosticScalingMixin, OutputScalingMixin): - - def __call__(self, state, timestep): - input = self.apply_input_scaling(state) - diagnostics, output = self.wrapped_component(input, timestep) - diagnostics = self.apply_diagnostic_scaling(diagnostics) - output = self.apply_output_scaling(output) - return diagnostics, output - - class ScalingWrapper(object): """ Wraps any component and scales either inputs, outputs or tendencies by a floating point value. Example ------- - This is how the ScaledInputOutputWrapper can be used to wrap a Prognostic. - >>> scaled_component = ScaledInputOutputWrapper( + This is how the ScalingWrapper can be used to wrap a Prognostic. + >>> scaled_component = ScalingWrapper( >>> RRTMRadiation(), >>> input_scale_factors = { >>> 'specific_humidity' = 0.2}, @@ -232,12 +54,20 @@ def __init__(self, The keys in the scale factors do not correspond to valid input/output/tendency for this component. """ + if not any( + isinstance(component, t) for t in [ + Diagnostic, Prognostic, ImplicitPrognostic, Implicit]): + raise TypeError( + 'component must be a component type (Diagnostic, Prognostic, ' + 'ImplicitPrognostic, or Implicit)' + ) + self._component = component self._input_scale_factors = dict() if input_scale_factors is not None: for input_field in input_scale_factors.keys(): - if input_field not in component.inputs: + if input_field not in component.input_properties.keys(): raise ValueError( "{} is not a valid input quantity.".format(input_field)) @@ -245,98 +75,113 @@ def __init__(self, self._diagnostic_scale_factors = dict() if diagnostic_scale_factors is not None: - - for diagnostic_field in diagnostic_scale_factors.keys(): - if diagnostic_field not in component.diagnostics: - raise ValueError( - "{} is not a valid diagnostic quantity.".format(diagnostic_field)) - + if not hasattr(component, 'diagnostic_properties'): + raise TypeError( + 'Cannot apply diagnostic scale factors to component without ' + 'diagnostic output.') + self._ensure_fields_have_properties( + diagnostic_scale_factors, component.diagnostic_properties, 'diagnostic') self._diagnostic_scale_factors = diagnostic_scale_factors - if hasattr(component, 'input_properties') and hasattr(component, 'output_properties'): - - self._output_scale_factors = dict() - if output_scale_factors is not None: - - for output_field in output_scale_factors.keys(): - if output_field not in component.outputs: - raise ValueError( - "{} is not a valid output quantity.".format(output_field)) - - self._output_scale_factors = output_scale_factors - self._component_type = 'Implicit' - - elif hasattr(component, 'input_properties') and hasattr(component, 'tendency_properties'): - - self._tendency_scale_factors = dict() - if tendency_scale_factors is not None: - - for tendency_field in tendency_scale_factors.keys(): - if tendency_field not in component.tendencies: - raise ValueError( - "{} is not a valid tendency quantity.".format(tendency_field)) - - self._tendency_scale_factors = tendency_scale_factors - self._component_type = 'Prognostic' - - elif hasattr(component, 'input_properties') and hasattr(component, 'diagnostic_properties'): - self._component_type = 'Diagnostic' - else: - raise TypeError( - "Component must be either of type Implicit or Prognostic or Diagnostic") - - self._component = component + self._output_scale_factors = dict() + if output_scale_factors is not None: + if not hasattr(component, 'output_properties'): + raise TypeError( + 'Cannot apply output scale factors to component without ' + 'output_properties.') + self._ensure_fields_have_properties( + output_scale_factors, component.output_properties, 'output') + self._output_scale_factors = output_scale_factors + + self._tendency_scale_factors = dict() + if tendency_scale_factors is not None: + if not hasattr(component, 'tendency_properties'): + raise TypeError( + 'Cannot apply tendency scale factors to component that does ' + 'not output tendencies.') + self._ensure_fields_have_properties( + tendency_scale_factors, component.tendency_properties, 'tendency') + self._tendency_scale_factors = tendency_scale_factors + + def _ensure_fields_have_properties( + self, scale_factors, properties, properties_name): + for field in scale_factors.keys(): + if field not in properties.keys(): + raise ValueError( + "{} is not a {} quantity in the given component" + ", but was given a scale factor.".format(field, properties_name)) def __getattr__(self, item): return getattr(self._component, item) def __call__(self, state, timestep=None): + """ + Call the underlying component, applying scaling. + + Parameters + ---------- + state : dict + A model state dictionary. + timestep : timedelta, optional + A time step. If the underlying component does not use a timestep, + this will be discarded. If it does, this argument is required. + Returns + ------- + *args + The return values of the underlying component. + """ scaled_state = {} if 'time' in state: scaled_state['time'] = state['time'] - for input_field in self.inputs: + for input_field in self.input_properties.keys(): if input_field in self._input_scale_factors: scale_factor = self._input_scale_factors[input_field] scaled_state[input_field] = state[input_field]*float(scale_factor) + scaled_state[input_field].attrs = state[input_field].attrs else: scaled_state[input_field] = state[input_field] - if self._component_type == 'Implicit': + if isinstance(self._component, Implicit): + if timestep is None: + raise TypeError('Must give timestep to call Implicit.') diagnostics, new_state = self._component(scaled_state, timestep) - - for output_field in self._output_scale_factors.keys(): - scale_factor = self._output_scale_factors[output_field] - new_state[output_field] *= float(scale_factor) - - for diagnostic_field in self._diagnostic_scale_factors.keys(): - scale_factor = self._diagnostic_scale_factors[diagnostic_field] - diagnostics[diagnostic_field] *= float(scale_factor) - + for name in self._output_scale_factors.keys(): + scale_factor = self._output_scale_factors[name] + new_state[name] *= float(scale_factor) + for name in self._diagnostic_scale_factors.keys(): + scale_factor = self._diagnostic_scale_factors[name] + diagnostics[name] *= float(scale_factor) return diagnostics, new_state - elif self._component_type == 'Prognostic': + elif isinstance(self._component, Prognostic): tendencies, diagnostics = self._component(scaled_state) - for tend_field in self._tendency_scale_factors.keys(): scale_factor = self._tendency_scale_factors[tend_field] tendencies[tend_field] *= float(scale_factor) - - for diagnostic_field in self._diagnostic_scale_factors.keys(): - scale_factor = self._diagnostic_scale_factors[diagnostic_field] - diagnostics[diagnostic_field] *= float(scale_factor) - + for name in self._diagnostic_scale_factors.keys(): + scale_factor = self._diagnostic_scale_factors[name] + diagnostics[name] *= float(scale_factor) return tendencies, diagnostics - elif self._component_type == 'Diagnostic': + elif isinstance(self._component, ImplicitPrognostic): + if timestep is None: + raise TypeError('Must give timestep to call ImplicitPrognostic.') + tendencies, diagnostics = self._component(scaled_state, timestep) + for tend_field in self._tendency_scale_factors.keys(): + scale_factor = self._tendency_scale_factors[tend_field] + tendencies[tend_field] *= float(scale_factor) + for name in self._diagnostic_scale_factors.keys(): + scale_factor = self._diagnostic_scale_factors[name] + diagnostics[name] *= float(scale_factor) + return tendencies, diagnostics + elif isinstance(self._component, Diagnostic): diagnostics = self._component(scaled_state) - - for diagnostic_field in self._diagnostic_scale_factors.keys(): - scale_factor = self._diagnostic_scale_factors[diagnostic_field] - diagnostics[diagnostic_field] *= float(scale_factor) - + for name in self._diagnostic_scale_factors.keys(): + scale_factor = self._diagnostic_scale_factors[name] + diagnostics[name] *= float(scale_factor) return diagnostics else: # Should never reach this - raise ValueError( + raise RuntimeError( 'Unknown component type, seems to be a bug in ScalingWrapper') @@ -354,29 +199,32 @@ class UpdateFrequencyWrapper(object): >>> prognostic = UpdateFrequencyWrapper(MyPrognostic(), timedelta(hours=1)) """ - def __init__(self, prognostic, update_timedelta): + def __init__(self, component, update_timedelta): """ Initialize the UpdateFrequencyWrapper object. Args ---- - prognostic : Prognostic - The object to be wrapped. + component + The component object to be wrapped. update_timedelta : timedelta The amount that state['time'] must differ from when output was cached before new output is computed. """ - self._prognostic = prognostic + self.component = component self._update_timedelta = update_timedelta self._cached_output = None self._last_update_time = None - def __call__(self, state, **kwargs): + def __call__(self, state, timestep=None, **kwargs): if ((self._last_update_time is None) or (state['time'] >= self._last_update_time + self._update_timedelta)): - self._cached_output = self._prognostic(state, **kwargs) + if timestep is not None: + self._cached_output = self.component(state, timestep, **kwargs) + else: + self._cached_output = self.component(state, **kwargs) self._last_update_time = state['time'] return self._cached_output def __getattr__(self, item): - return getattr(self._prognostic, item) + return getattr(self.component, item) diff --git a/tests/test_base_components.py b/tests/test_base_components.py index a1f60f5..34aac99 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -25,8 +25,8 @@ def __init__( self.input_properties = input_properties self.diagnostic_properties = diagnostic_properties self.tendency_properties = tendency_properties - self._diagnostic_output = diagnostic_output - self._tendency_output = tendency_output + self.diagnostic_output = diagnostic_output + self.tendency_output = tendency_output self.times_called = 0 self.state_given = None super(MockPrognostic, self).__init__(**kwargs) @@ -34,7 +34,7 @@ def __init__( def array_call(self, state): self.times_called += 1 self.state_given = state - return self._tendency_output, self._diagnostic_output + return self.tendency_output, self.diagnostic_output class MockImplicitPrognostic(ImplicitPrognostic): @@ -49,8 +49,8 @@ def __init__( self.input_properties = input_properties self.diagnostic_properties = diagnostic_properties self.tendency_properties = tendency_properties - self._diagnostic_output = diagnostic_output - self._tendency_output = tendency_output + self.diagnostic_output = diagnostic_output + self.tendency_output = tendency_output self.times_called = 0 self.state_given = None self.timestep_given = None @@ -60,7 +60,7 @@ def array_call(self, state, timestep): self.times_called += 1 self.state_given = state self.timestep_given = timestep - return self._tendency_output, self._diagnostic_output + return self.tendency_output, self.diagnostic_output class MockDiagnostic(Diagnostic): @@ -73,7 +73,7 @@ def __init__( **kwargs): self.input_properties = input_properties self.diagnostic_properties = diagnostic_properties - self._diagnostic_output = diagnostic_output + self.diagnostic_output = diagnostic_output self.times_called = 0 self.state_given = None super(MockDiagnostic, self).__init__(**kwargs) @@ -81,7 +81,7 @@ def __init__( def array_call(self, state): self.times_called += 1 self.state_given = state - return self._diagnostic_output + return self.diagnostic_output class MockImplicit(Implicit): @@ -97,8 +97,8 @@ def __init__( self.input_properties = input_properties self.diagnostic_properties = diagnostic_properties self.output_properties = output_properties - self._diagnostic_output = diagnostic_output - self._state_output = state_output + self.diagnostic_output = diagnostic_output + self.state_output = state_output self.times_called = 0 self.state_given = None self.timestep_given = None @@ -108,7 +108,7 @@ def array_call(self, state, timestep): self.times_called += 1 self.state_given = state self.timestep_given = timestep - return self._diagnostic_output, self._state_output + return self.diagnostic_output, self.state_output class MockMonitor(Monitor): @@ -124,6 +124,69 @@ class PrognosticTests(unittest.TestCase): def call_component(self, component, state): return component(state) + def test_subclass_check(self): + class MyPrognostic(object): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + tendencies_in_diagnostics = False + name = '' + def __call__(self): + pass + def array_call(self): + pass + + instance = MyPrognostic() + assert isinstance(instance, Prognostic) + + def test_two_components_are_not_instances_of_each_other(self): + class MyPrognostic1(Prognostic): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + tendencies_in_diagnostics = False + name = '' + def array_call(self, state): + pass + + class MyPrognostic2(Prognostic): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + tendencies_in_diagnostics = False + name = '' + def array_call(self, state): + pass + + prog1 = MyPrognostic1() + prog2 = MyPrognostic2() + assert not isinstance(prog1, MyPrognostic2) + assert not isinstance(prog2, MyPrognostic1) + + def test_ducktype_not_instance_of_subclass(self): + class MyPrognostic1(object): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + tendencies_in_diagnostics = False + name = '' + def __init__(self): + pass + def array_call(self, state): + pass + + class MyPrognostic2(Prognostic): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + tendencies_in_diagnostics = False + name = '' + def array_call(self, state): + pass + + prog1 = MyPrognostic1() + assert not isinstance(prog1, MyPrognostic2) + def test_empty_prognostic(self): prognostic = self.component_class({}, {}, {}, {}, {}) tendencies, diagnostics = self.call_component( @@ -160,9 +223,99 @@ def test_input_requires_units(self): diagnostic_output, tendency_output ) + def test_diagnostic_requires_dims(self): + input_properties = {} + diagnostic_properties = {'diag1': {'units': 'm'}} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + + def test_diagnostic_requires_correct_number_of_dims(self): + input_properties = { + 'input1': {'units': 'm', 'dims': ['dim1', 'dim2']} + } + diagnostic_properties = { + 'diag1': {'units': 'm', 'dims': ['dim1', 'dim2']} + } + tendency_properties = {} + diagnostic_output = {'diag1': np.zeros([10]),} + tendency_output = {} + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10, 2]), + dims=['dim1', 'dim2'], + attrs={'units': 'm'} + ) + } + component = self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + with self.assertRaises(InvalidPropertyDictError): + _, _ = self.call_component(component, state) + + def test_diagnostic_requires_correct_dim_length(self): + input_properties = { + 'input1': {'units': 'm', 'dims': ['dim1', 'dim2']} + } + diagnostic_properties = { + 'diag1': {'units': 'm', 'dims': ['dim1', 'dim2']} + } + tendency_properties = {} + diagnostic_output = {'diag1': np.zeros([5, 2]),} + tendency_output = {} + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10, 2]), + dims=['dim1', 'dim2'], + attrs={'units': 'm'} + ) + } + component = self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + with self.assertRaises(InvalidPropertyDictError): + _, _ = self.call_component(component, state) + + def test_diagnostic_uses_base_dims(self): + input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} + diagnostic_properties = {'diag1': {'units': 'm'}} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + + def test_diagnostic_doesnt_use_base_units(self): + input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} + diagnostic_properties = {'diag1': {'dims': ['dim1']}} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + def test_diagnostic_requires_units(self): input_properties = {} - diagnostic_properties = {'diag1': {}} + diagnostic_properties = {'diag1': {'dims': ['dim1']}} tendency_properties = {} diagnostic_output = {} tendency_output = {} @@ -173,10 +326,48 @@ def test_diagnostic_requires_units(self): diagnostic_output, tendency_output ) + def test_tendency_requires_dims(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {'tend1': {'units': 'm'}} + diagnostic_output = {} + tendency_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + + def test_tendency_uses_base_dims(self): + input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} + diagnostic_properties = {} + tendency_properties = {'diag1': {'units': 'm'}} + diagnostic_output = {} + tendency_output = {} + self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + + def test_tendency_doesnt_use_base_units(self): + input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} + diagnostic_properties = {} + tendency_properties = {'diag1': {'dims': ['dim1']}} + diagnostic_output = {} + tendency_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + def test_tendency_requires_units(self): input_properties = {} diagnostic_properties = {} - tendency_properties = {'tend1': {}} + tendency_properties = {'tend1': {'dims': ['dim1']}} diagnostic_output = {} tendency_output = {} with self.assertRaises(InvalidPropertyDictError): @@ -205,6 +396,57 @@ def test_raises_when_tendency_not_given(self): with self.assertRaises(ComponentMissingOutputError): _, _ = self.call_component(prognostic, state) + def test_cannot_overlap_input_aliases(self): + input_properties = { + 'input1': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'}, + 'input2': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'} + } + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + + def test_cannot_overlap_diagnostic_aliases(self): + input_properties = { + } + diagnostic_properties = { + 'diag1': {'dims': ['dim1'], 'units': 'm', 'alias': 'diag'}, + 'diag2': {'dims': ['dim1'], 'units': 'm', 'alias': 'diag'} + } + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + + def test_cannot_overlap_tendency_aliases(self): + input_properties = { + } + diagnostic_properties = { + } + tendency_properties = { + 'tend1': {'dims': ['dim1'], 'units': 'm', 'alias': 'tend'}, + 'tend2': {'dims': ['dim1'], 'units': 'm', 'alias': 'tend'} + } + diagnostic_output = {} + tendency_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output + ) + def test_raises_when_extraneous_tendency_given(self): input_properties = {} diagnostic_properties = {} @@ -540,25 +782,34 @@ def test_diagnostics_no_transformations(self): assert diagnostics['output1'].attrs['units'] == 'm' assert np.all(diagnostics['output1'].values == np.ones([10])) - def test_diagnostics_with_alias(self): - input_properties = {} + def test_diagnostics_restoring_dims(self): + input_properties = { + 'input1': { + 'dims': ['*', 'dim1'], + 'units': 'm', + } + } diagnostic_properties = { 'output1': { - 'dims': ['dim1'], - 'units': 'm', - 'alias': 'out1', + 'dims': ['*', 'dim1'], + 'units': 'm' } } tendency_properties = {} diagnostic_output = { - 'out1': np.ones([10]), + 'output1': np.ones([1, 10]), } tendency_output = {} prognostic = self.component_class( input_properties, diagnostic_properties, tendency_properties, diagnostic_output, tendency_output ) - state = {'time': timedelta(0)} + state = { + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'}), + 'time': timedelta(0)} _, diagnostics = self.call_component(prognostic, state) assert len(diagnostics) == 1 assert 'output1' in diagnostics.keys() @@ -569,18 +820,13 @@ def test_diagnostics_with_alias(self): assert diagnostics['output1'].attrs['units'] == 'm' assert np.all(diagnostics['output1'].values == np.ones([10])) - def test_diagnostics_with_alias_from_input(self): - input_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - 'alias': 'out1', - } - } + def test_diagnostics_with_alias(self): + input_properties = {} diagnostic_properties = { 'output1': { 'dims': ['dim1'], 'units': 'm', + 'alias': 'out1', } } tendency_properties = {} @@ -592,14 +838,7 @@ def test_diagnostics_with_alias_from_input(self): input_properties, diagnostic_properties, tendency_properties, diagnostic_output, tendency_output ) - state = { - 'time': timedelta(0), - 'output1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } + state = {'time': timedelta(0)} _, diagnostics = self.call_component(prognostic, state) assert len(diagnostics) == 1 assert 'output1' in diagnostics.keys() @@ -610,21 +849,23 @@ def test_diagnostics_with_alias_from_input(self): assert diagnostics['output1'].attrs['units'] == 'm' assert np.all(diagnostics['output1'].values == np.ones([10])) - def test_diagnostics_with_dims_from_input(self): + def test_diagnostics_with_alias_from_input(self): input_properties = { 'output1': { 'dims': ['dim1'], 'units': 'm', + 'alias': 'out1', } } diagnostic_properties = { 'output1': { + 'dims': ['dim1'], 'units': 'm', } } tendency_properties = {} diagnostic_output = { - 'output1': np.ones([10]), + 'out1': np.ones([10]), } tendency_output = {} prognostic = self.component_class( @@ -649,85 +890,16 @@ def test_diagnostics_with_dims_from_input(self): assert diagnostics['output1'].attrs['units'] == 'm' assert np.all(diagnostics['output1'].values == np.ones([10])) - def test_input_scaling(self): - input_scale_factors = {'input1': 2.} + def test_diagnostics_with_dims_from_input(self): input_properties = { - 'input1': { - 'dims': ['dim1'], - 'units': 'm' - } - } - diagnostic_properties = {} - tendency_properties = {} - diagnostic_output = {} - tendency_output = {} - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output, - input_scale_factors=input_scale_factors - ) - assert prognostic.tendency_scale_factors == {} - assert prognostic.diagnostic_scale_factors == {} - assert len(prognostic.input_scale_factors) == 1 - assert 'input1' in prognostic.input_scale_factors.keys() - assert prognostic.input_scale_factors['input1'] == 2. - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - _, _ = self.call_component(prognostic, state) - assert len(prognostic.state_given) == 2 - assert 'time' in prognostic.state_given.keys() - assert 'input1' in prognostic.state_given.keys() - assert isinstance(prognostic.state_given['input1'], np.ndarray) - assert np.all(prognostic.state_given['input1'] == np.ones([10])*2.) - - def test_tendency_scaling(self): - tendency_scale_factors = {'output1': 3.} - input_properties = {} - diagnostic_properties = {} - tendency_properties = { 'output1': { 'dims': ['dim1'], - 'units': 'm/s' + 'units': 'm', } } - diagnostic_output = {} - tendency_output = { - 'output1': np.ones([10]), - } - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output, - tendency_scale_factors=tendency_scale_factors, - ) - assert prognostic.input_scale_factors == {} - assert prognostic.diagnostic_scale_factors == {} - assert len(prognostic.tendency_scale_factors) == 1 - assert 'output1' in prognostic.tendency_scale_factors.keys() - assert prognostic.tendency_scale_factors['output1'] == 3. - state = {'time': timedelta(0)} - tendencies, _ = self.call_component(prognostic, state) - assert len(tendencies) == 1 - assert 'output1' in tendencies.keys() - assert isinstance(tendencies['output1'], DataArray) - assert len(tendencies['output1'].dims) == 1 - assert 'dim1' in tendencies['output1'].dims - assert 'units' in tendencies['output1'].attrs - assert tendencies['output1'].attrs['units'] == 'm/s' - assert np.all(tendencies['output1'].values == np.ones([10])*3.) - - def test_diagnostics_scaling(self): - diagnostic_scale_factors = {'output1': 0.} - input_properties = {} diagnostic_properties = { 'output1': { - 'dims': ['dim1'], - 'units': 'm' + 'units': 'm', } } tendency_properties = {} @@ -737,15 +909,16 @@ def test_diagnostics_scaling(self): tendency_output = {} prognostic = self.component_class( input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output, - diagnostic_scale_factors=diagnostic_scale_factors, + diagnostic_output, tendency_output ) - assert prognostic.tendency_scale_factors == {} - assert prognostic.input_scale_factors == {} - assert len(prognostic.diagnostic_scale_factors) == 1 - assert 'output1' in prognostic.diagnostic_scale_factors.keys() - assert prognostic.diagnostic_scale_factors['output1'] == 0. - state = {'time': timedelta(0)} + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } _, diagnostics = self.call_component(prognostic, state) assert len(diagnostics) == 1 assert 'output1' in diagnostics.keys() @@ -754,76 +927,7 @@ def test_diagnostics_scaling(self): assert 'dim1' in diagnostics['output1'].dims assert 'units' in diagnostics['output1'].attrs assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])*0.) - - def test_update_interval_on_timedelta(self): - input_properties = {} - diagnostic_properties = {} - tendency_properties = {} - diagnostic_output = {} - tendency_output = {} - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output, update_interval=timedelta(seconds=10) - ) - _, _ = self.call_component(prognostic, {'time': timedelta(seconds=0)}) - assert prognostic.times_called == 1 - _, _ = self.call_component(prognostic, {'time': timedelta(seconds=0)}) - assert prognostic.times_called == 1, 'should not re-compute output' - _, _ = self.call_component(prognostic, {'time': timedelta(seconds=5)}) - assert prognostic.times_called == 1, 'should not re-compute output' - _, _ = self.call_component(prognostic, {'time': timedelta(seconds=10)}) - assert prognostic.times_called == 2, 'should re-compute output' - _, _ = self.call_component(prognostic, {'time': timedelta(seconds=15)}) - assert prognostic.times_called == 2, 'should not re-compute output' - _, _ = self.call_component(prognostic, {'time': timedelta(seconds=20)}) - assert prognostic.times_called == 3, 'should re-compute output' - _, _ = self.call_component(prognostic, {'time': timedelta(seconds=30)}) - assert prognostic.times_called == 4, 'should re-compute output' - _, _ = self.call_component(prognostic, {'time': timedelta(seconds=45)}) - assert prognostic.times_called == 5, 'should re-compute output' - _, _ = self.call_component(prognostic, {'time': timedelta(seconds=50)}) - assert prognostic.times_called == 5, 'should not re-compute output' - - def test_update_interval_on_datetime(self): - input_properties = {} - diagnostic_properties = {} - tendency_properties = {} - diagnostic_output = {} - tendency_output = {} - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output, - update_interval=timedelta(seconds=10) - ) - dt = datetime(2010, 1, 1) - _, _ = self.call_component( - prognostic, {'time': dt + timedelta(seconds=0)}) - assert prognostic.times_called == 1 - _, _ = self.call_component( - prognostic, {'time': dt + timedelta(seconds=0)}) - assert prognostic.times_called == 1, 'should not re-compute output' - _, _ = self.call_component( - prognostic, {'time': dt + timedelta(seconds=5)}) - assert prognostic.times_called == 1, 'should not re-compute output' - _, _ = self.call_component( - prognostic, {'time': dt + timedelta(seconds=10)}) - assert prognostic.times_called == 2, 'should re-compute output' - _, _ = self.call_component( - prognostic, {'time': dt + timedelta(seconds=15)}) - assert prognostic.times_called == 2, 'should not re-compute output' - _, _ = self.call_component( - prognostic, {'time': dt + timedelta(seconds=20)}) - assert prognostic.times_called == 3, 'should re-compute output' - _, _ = self.call_component( - prognostic, {'time': dt + timedelta(seconds=30)}) - assert prognostic.times_called == 4, 'should re-compute output' - _, _ = self.call_component( - prognostic, {'time': dt + timedelta(seconds=45)}) - assert prognostic.times_called == 5, 'should re-compute output' - _, _ = self.call_component( - prognostic, {'time': dt + timedelta(seconds=50)}) - assert prognostic.times_called == 5, 'should not re-compute output' + assert np.all(diagnostics['output1'].values == np.ones([10])) def test_tendencies_in_diagnostics_no_tendency(self): input_properties = {} @@ -962,6 +1066,100 @@ class ImplicitPrognosticTests(PrognosticTests): def call_component(self, component, state): return component(state, timedelta(seconds=1)) + def test_subclass_check(self): + class MyImplicitPrognostic(object): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + tendencies_in_diagnostics = False + name = '' + def __call__(self, state, timestep): + pass + def array_call(self, state, timestep): + pass + + instance = MyImplicitPrognostic() + assert isinstance(instance, ImplicitPrognostic) + + def test_two_components_are_not_instances_of_each_other(self): + class MyImplicitPrognostic1(ImplicitPrognostic): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + tendencies_in_diagnostics = False + name = '' + + def array_call(self, state, timestep): + pass + + class MyImplicitPrognostic2(ImplicitPrognostic): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + tendencies_in_diagnostics = False + name = '' + + def array_call(self, state): + pass + + prog1 = MyImplicitPrognostic1() + prog2 = MyImplicitPrognostic2() + assert not isinstance(prog1, MyImplicitPrognostic2) + assert not isinstance(prog2, MyImplicitPrognostic1) + + def test_ducktype_not_instance_of_subclass(self): + class MyImplicitPrognostic1(object): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + tendencies_in_diagnostics = False + name = '' + def __init__(self): + pass + def array_call(self, state, timestep): + pass + + class MyImplicitPrognostic2(ImplicitPrognostic): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + tendencies_in_diagnostics = False + name = '' + + def array_call(self, state): + pass + + prog1 = MyImplicitPrognostic1() + assert not isinstance(prog1, MyImplicitPrognostic2) + + def test_subclass_is_not_prognostic(self): + class MyImplicitPrognostic(ImplicitPrognostic): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + tendencies_in_diagnostics = False + name = '' + def array_call(self, state, timestep): + pass + + instance = MyImplicitPrognostic() + assert not isinstance(instance, Prognostic) + + def test_ducktype_is_not_prognostic(self): + class MyImplicitPrognostic(object): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + tendencies_in_diagnostics = False + name = '' + def __call__(self, state, timestep): + pass + def array_call(self, state, timestep): + pass + + instance = MyImplicitPrognostic() + assert not isinstance(instance, Prognostic) + def test_timedelta_is_passed(self): prognostic = MockImplicitPrognostic({}, {}, {}, {}, {}) tendencies, diagnostics = prognostic( @@ -979,6 +1177,57 @@ class DiagnosticTests(unittest.TestCase): def call_component(self, component, state): return component(state) + def test_subclass_check(self): + class MyDiagnostic(object): + input_properties = {} + diagnostic_properties = {} + def __call__(self, state): + pass + def array_call(self, state): + pass + + instance = MyDiagnostic() + assert isinstance(instance, Diagnostic) + + def test_two_components_are_not_instances_of_each_other(self): + class MyDiagnostic1(Diagnostic): + input_properties = {} + diagnostic_properties = {} + + def array_call(self, state): + pass + + class MyDiagnostic2(Diagnostic): + input_properties = {} + diagnostic_properties = {} + + def array_call(self, state): + pass + + diag1 = MyDiagnostic1() + diag2 = MyDiagnostic2() + assert not isinstance(diag1, MyDiagnostic2) + assert not isinstance(diag2, MyDiagnostic1) + + def test_ducktype_not_instance_of_subclass(self): + class MyDiagnostic1(object): + input_properties = {} + diagnostic_properties = {} + def __init__(self): + pass + def array_call(self, state): + pass + + class MyDiagnostic2(Diagnostic): + input_properties = {} + diagnostic_properties = {} + + def array_call(self, state): + pass + + diag1 = MyDiagnostic1() + assert not isinstance(diag1, MyDiagnostic2) + def test_empty_diagnostic(self): diagnostic = self.component_class({}, {}, {}) diagnostics = diagnostic({'time': timedelta(seconds=0)}) @@ -1008,9 +1257,38 @@ def test_input_requires_units(self): diagnostic_output ) + def test_diagnostic_requires_dims(self): + input_properties = {} + diagnostic_properties = {'diag1': {'units': 'm'}} + diagnostic_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + diagnostic_output, + ) + + def test_diagnostic_uses_base_dims(self): + input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} + diagnostic_properties = {'diag1': {'units': 'm'}} + diagnostic_output = {} + self.component_class( + input_properties, diagnostic_properties, + diagnostic_output, + ) + + def test_diagnostic_doesnt_use_base_units(self): + input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} + diagnostic_properties = {'diag1': {'dims': ['dim1']}} + diagnostic_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + diagnostic_output, + ) + def test_diagnostic_requires_units(self): input_properties = {} - diagnostic_properties = {'diag1': {}} + diagnostic_properties = {'diag1': {'dims': ['dim1']}} diagnostic_output = {} with self.assertRaises(InvalidPropertyDictError): self.component_class( @@ -1018,6 +1296,33 @@ def test_diagnostic_requires_units(self): diagnostic_output, ) + def test_cannot_overlap_input_aliases(self): + input_properties = { + 'input1': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'}, + 'input2': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'} + } + diagnostic_properties = {} + diagnostic_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + diagnostic_output + ) + + def test_cannot_overlap_diagnostic_aliases(self): + input_properties = { + } + diagnostic_properties = { + 'diag1': {'dims': ['dim1'], 'units': 'm', 'alias': 'diag'}, + 'diag2': {'dims': ['dim1'], 'units': 'm', 'alias': 'diag'} + } + diagnostic_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + diagnostic_output + ) + def test_raises_when_diagnostic_not_given(self): input_properties = {} diagnostic_properties = { @@ -1219,54 +1524,17 @@ def test_diagnostics_with_alias_from_input(self): 'output1': { 'dims': ['dim1'], 'units': 'm', - 'alias': 'out1', - } - } - diagnostic_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - } - } - diagnostic_output = { - 'out1': np.ones([10]), - } - diagnostic = MockDiagnostic( - input_properties, diagnostic_properties, - diagnostic_output - ) - state = { - 'time': timedelta(0), - 'output1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - diagnostics = diagnostic(state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) - - def test_diagnostics_with_dims_from_input(self): - input_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', + 'alias': 'out1', } } diagnostic_properties = { 'output1': { + 'dims': ['dim1'], 'units': 'm', } } diagnostic_output = { - 'output1': np.ones([10]), + 'out1': np.ones([10]), } diagnostic = MockDiagnostic( input_properties, diagnostic_properties, @@ -1290,63 +1558,34 @@ def test_diagnostics_with_dims_from_input(self): assert diagnostics['output1'].attrs['units'] == 'm' assert np.all(diagnostics['output1'].values == np.ones([10])) - def test_input_scaling(self): - input_scale_factors = {'input1': 2.} + def test_diagnostics_with_dims_from_input(self): input_properties = { - 'input1': { + 'output1': { 'dims': ['dim1'], - 'units': 'm' + 'units': 'm', } } - diagnostic_properties = {} - diagnostic_output = {} - diagnostic = self.component_class( - input_properties, diagnostic_properties, - diagnostic_output, - input_scale_factors=input_scale_factors - ) - assert diagnostic.diagnostic_scale_factors == {} - assert len(diagnostic.input_scale_factors) == 1 - assert 'input1' in diagnostic.input_scale_factors.keys() - assert diagnostic.input_scale_factors['input1'] == 2. - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - _ = self.call_component(diagnostic, state) - assert len(diagnostic.state_given) == 2 - assert 'time' in diagnostic.state_given.keys() - assert 'input1' in diagnostic.state_given.keys() - assert isinstance(diagnostic.state_given['input1'], np.ndarray) - assert np.all(diagnostic.state_given['input1'] == np.ones([10]) * 2.) - - def test_diagnostics_scaling(self): - diagnostic_scale_factors = {'output1': 0.} - input_properties = {} diagnostic_properties = { 'output1': { - 'dims': ['dim1'], - 'units': 'm' + 'units': 'm', } } diagnostic_output = { 'output1': np.ones([10]), } - diagnostic = self.component_class( + diagnostic = MockDiagnostic( input_properties, diagnostic_properties, - diagnostic_output, - diagnostic_scale_factors=diagnostic_scale_factors, + diagnostic_output ) - assert diagnostic.input_scale_factors == {} - assert len(diagnostic.diagnostic_scale_factors) == 1 - assert 'output1' in diagnostic.diagnostic_scale_factors.keys() - assert diagnostic.diagnostic_scale_factors['output1'] == 0. - state = {'time': timedelta(0)} - diagnostics = self.call_component(diagnostic, state) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + diagnostics = diagnostic(state) assert len(diagnostics) == 1 assert 'output1' in diagnostics.keys() assert isinstance(diagnostics['output1'], DataArray) @@ -1354,73 +1593,7 @@ def test_diagnostics_scaling(self): assert 'dim1' in diagnostics['output1'].dims assert 'units' in diagnostics['output1'].attrs assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10]) * 0.) - - def test_update_interval_on_timedelta(self): - input_properties = {} - diagnostic_properties = {} - diagnostic_output = {} - diagnostic = self.component_class( - input_properties, diagnostic_properties, - diagnostic_output, - update_interval=timedelta(seconds=10) - ) - _ = self.call_component(diagnostic, {'time': timedelta(seconds=0)}) - assert diagnostic.times_called == 1 - _ = self.call_component(diagnostic, {'time': timedelta(seconds=0)}) - assert diagnostic.times_called == 1, 'should not re-compute output' - _ = self.call_component(diagnostic, {'time': timedelta(seconds=5)}) - assert diagnostic.times_called == 1, 'should not re-compute output' - _ = self.call_component(diagnostic, {'time': timedelta(seconds=10)}) - assert diagnostic.times_called == 2, 'should re-compute output' - _ = self.call_component(diagnostic, {'time': timedelta(seconds=15)}) - assert diagnostic.times_called == 2, 'should not re-compute output' - _ = self.call_component(diagnostic, {'time': timedelta(seconds=20)}) - assert diagnostic.times_called == 3, 'should re-compute output' - _ = self.call_component(diagnostic, {'time': timedelta(seconds=30)}) - assert diagnostic.times_called == 4, 'should re-compute output' - _ = self.call_component(diagnostic, {'time': timedelta(seconds=45)}) - assert diagnostic.times_called == 5, 'should re-compute output' - _ = self.call_component(diagnostic, {'time': timedelta(seconds=50)}) - assert diagnostic.times_called == 5, 'should not re-compute output' - - def test_update_interval_on_datetime(self): - input_properties = {} - diagnostic_properties = {} - diagnostic_output = {} - diagnostic = self.component_class( - input_properties, diagnostic_properties, - diagnostic_output, - update_interval=timedelta(seconds=10) - ) - dt = datetime(2010, 1, 1) - _ = self.call_component( - diagnostic, {'time': dt + timedelta(seconds=0)}) - assert diagnostic.times_called == 1 - _ = self.call_component( - diagnostic, {'time': dt + timedelta(seconds=0)}) - assert diagnostic.times_called == 1, 'should not re-compute output' - _ = self.call_component( - diagnostic, {'time': dt + timedelta(seconds=5)}) - assert diagnostic.times_called == 1, 'should not re-compute output' - _ = self.call_component( - diagnostic, {'time': dt + timedelta(seconds=10)}) - assert diagnostic.times_called == 2, 'should re-compute output' - _ = self.call_component( - diagnostic, {'time': dt + timedelta(seconds=15)}) - assert diagnostic.times_called == 2, 'should not re-compute output' - _ = self.call_component( - diagnostic, {'time': dt + timedelta(seconds=20)}) - assert diagnostic.times_called == 3, 'should re-compute output' - _ = self.call_component( - diagnostic, {'time': dt + timedelta(seconds=30)}) - assert diagnostic.times_called == 4, 'should re-compute output' - _ = self.call_component( - diagnostic, {'time': dt + timedelta(seconds=45)}) - assert diagnostic.times_called == 5, 'should re-compute output' - _ = self.call_component( - diagnostic, {'time': dt + timedelta(seconds=50)}) - assert diagnostic.times_called == 5, 'should not re-compute output' + assert np.all(diagnostics['output1'].values == np.ones([10])) class ImplicitTests(unittest.TestCase): @@ -1430,6 +1603,72 @@ class ImplicitTests(unittest.TestCase): def call_component(self, component, state): return component(state, timedelta(seconds=1)) + def test_subclass_check(self): + class MyImplicit(object): + input_properties = {} + diagnostic_properties = {} + output_properties = {} + tendencies_in_diagnostics = False + name = '' + def __call__(self, state, timestep): + pass + def array_call(self, state, timestep): + pass + + instance = MyImplicit() + assert isinstance(instance, Implicit) + + def test_two_components_are_not_instances_of_each_other(self): + class MyImplicit1(Implicit): + input_properties = {} + diagnostic_properties = {} + output_properties = {} + tendencies_in_diagnostics = False + name = '' + + def array_call(self, state): + pass + + class MyImplicit2(Implicit): + input_properties = {} + diagnostic_properties = {} + output_properties = {} + tendencies_in_diagnostics = False + name = '' + + def array_call(self, state): + pass + + implicit1 = MyImplicit1() + implicit2 = MyImplicit2() + assert not isinstance(implicit1, MyImplicit2) + assert not isinstance(implicit2, MyImplicit1) + + def test_ducktype_not_instance_of_subclass(self): + class MyImplicit1(object): + input_properties = {} + diagnostic_properties = {} + output_properties = {} + tendencies_in_diagnostics = False + name = '' + def __init__(self): + pass + def array_call(self, state): + pass + + class MyImplicit2(Implicit): + input_properties = {} + diagnostic_properties = {} + output_properties = {} + tendencies_in_diagnostics = False + name = '' + + def array_call(self, state): + pass + + implicit1 = MyImplicit1() + assert not isinstance(implicit1, MyImplicit2) + def test_empty_implicit(self): implicit = self.component_class( {}, {}, {}, {}, {}) @@ -1468,9 +1707,47 @@ def test_input_requires_units(self): diagnostic_output, state_output ) + def test_diagnostic_requires_dims(self): + input_properties = {} + diagnostic_properties = {'diag1': {'units': 'm'}} + output_properties = {} + diagnostic_output = {} + state_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, state_output + ) + + def test_diagnostic_uses_base_dims(self): + input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} + diagnostic_properties = {'diag1': {'units': 'm'}} + output_properties = {} + diagnostic_output = {} + state_output = {} + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, state_output + ) + + def test_diagnostic_doesnt_use_base_units(self): + input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} + diagnostic_properties = {'diag1': {'dims': ['dim1']}} + output_properties = {} + diagnostic_output = {} + state_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, state_output + ) + def test_diagnostic_requires_units(self): input_properties = {} - diagnostic_properties = {'diag1': {}} + diagnostic_properties = {'diag1': {'dims': ['dim1']}} output_properties = {} diagnostic_output = {} state_output = {} @@ -1481,10 +1758,48 @@ def test_diagnostic_requires_units(self): diagnostic_output, state_output ) + def test_output_requires_dims(self): + input_properties = {} + diagnostic_properties = {} + output_properties = {'diag1': {'units': 'm'}} + diagnostic_output = {} + state_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, state_output + ) + + def test_output_uses_base_dims(self): + input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} + diagnostic_properties = {} + output_properties = {'diag1': {'units': 'm'}} + diagnostic_output = {} + state_output = {} + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, state_output + ) + + def test_output_doesnt_use_base_units(self): + input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} + diagnostic_properties = {} + output_properties = {'diag1': {'dims': ['dim1']}} + diagnostic_output = {} + state_output = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, state_output + ) + def test_output_requires_units(self): input_properties = {} diagnostic_properties = {} - output_properties = {'output1': {}} + output_properties = {'output1': {'dims': ['dim1']}} diagnostic_output = {} state_output = {} with self.assertRaises(InvalidPropertyDictError): @@ -1494,6 +1809,57 @@ def test_output_requires_units(self): diagnostic_output, state_output ) + def test_cannot_overlap_input_aliases(self): + input_properties = { + 'input1': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'}, + 'input2': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'} + } + diagnostic_properties = {} + output_properties = {} + diagnostic_output = {} + output_state = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, output_state + ) + + def test_cannot_overlap_diagnostic_aliases(self): + input_properties = { + } + diagnostic_properties = { + 'diag1': {'dims': ['dim1'], 'units': 'm', 'alias': 'diag'}, + 'diag2': {'dims': ['dim1'], 'units': 'm', 'alias': 'diag'} + } + output_properties = {} + diagnostic_output = {} + output_state = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, output_state + ) + + def test_cannot_overlap_output_aliases(self): + input_properties = { + } + diagnostic_properties = { + } + output_properties = { + 'out1': {'dims': ['dim1'], 'units': 'm', 'alias': 'out'}, + 'out2': {'dims': ['dim1'], 'units': 'm', 'alias': 'out'} + } + diagnostic_output = {} + output_state = {} + with self.assertRaises(InvalidPropertyDictError): + self.component_class( + input_properties, diagnostic_properties, + output_properties, + diagnostic_output, output_state + ) + def test_timedelta_is_passed(self): implicit = MockImplicit({}, {}, {}, {}, {}) tendencies, diagnostics = implicit( @@ -1966,183 +2332,6 @@ def test_diagnostics_with_dims_from_input(self): assert diagnostics['output1'].attrs['units'] == 'm' assert np.all(diagnostics['output1'].values == np.ones([10])) - def test_input_scaling(self): - input_scale_factors = {'input1': 2.} - input_properties = { - 'input1': { - 'dims': ['dim1'], - 'units': 'm' - } - } - diagnostic_properties = {} - output_properties = {} - diagnostic_output = {} - output_state = {} - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, output_state, - input_scale_factors=input_scale_factors - ) - assert implicit.output_scale_factors == {} - assert implicit.diagnostic_scale_factors == {} - assert len(implicit.input_scale_factors) == 1 - assert 'input1' in implicit.input_scale_factors.keys() - assert implicit.input_scale_factors['input1'] == 2. - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - _, _ = self.call_component(implicit, state) - assert len(implicit.state_given) == 2 - assert 'time' in implicit.state_given.keys() - assert 'input1' in implicit.state_given.keys() - assert isinstance(implicit.state_given['input1'], np.ndarray) - assert np.all(implicit.state_given['input1'] == np.ones([10]) * 2.) - - def test_output_scaling(self): - output_scale_factors = {'output1': 3.} - input_properties = {} - diagnostic_properties = {} - output_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm/s' - } - } - diagnostic_output = {} - output_state = { - 'output1': np.ones([10]), - } - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, output_state, - output_scale_factors=output_scale_factors, - ) - assert implicit.input_scale_factors == {} - assert implicit.diagnostic_scale_factors == {} - assert len(implicit.output_scale_factors) == 1 - assert 'output1' in implicit.output_scale_factors.keys() - assert implicit.output_scale_factors['output1'] == 3. - state = {'time': timedelta(0)} - _, output = self.call_component(implicit, state) - assert len(output) == 1 - assert 'output1' in output.keys() - assert isinstance(output['output1'], DataArray) - assert len(output['output1'].dims) == 1 - assert 'dim1' in output['output1'].dims - assert 'units' in output['output1'].attrs - assert output['output1'].attrs['units'] == 'm/s' - assert np.all(output['output1'].values == np.ones([10]) * 3.) - - def test_diagnostics_scaling(self): - diagnostic_scale_factors = {'output1': 0.} - input_properties = {} - diagnostic_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm' - } - } - output_properties = {} - diagnostic_output = { - 'output1': np.ones([10]), - } - tendency_output = {} - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, tendency_output, - diagnostic_scale_factors=diagnostic_scale_factors, - ) - assert implicit.output_scale_factors == {} - assert implicit.input_scale_factors == {} - assert len(implicit.diagnostic_scale_factors) == 1 - assert 'output1' in implicit.diagnostic_scale_factors.keys() - assert implicit.diagnostic_scale_factors['output1'] == 0. - state = {'time': timedelta(0)} - diagnostics, _ = self.call_component(implicit, state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10]) * 0.) - - def test_update_interval_on_timedelta(self): - input_properties = {} - diagnostic_properties = {} - output_properties = {} - diagnostic_output = {} - output_state = {} - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, output_state, - update_interval=timedelta(seconds=10) - ) - _, _ = self.call_component(implicit, {'time': timedelta(seconds=0)}) - assert implicit.times_called == 1 - _, _ = self.call_component(implicit, {'time': timedelta(seconds=0)}) - assert implicit.times_called == 1, 'should not re-compute output' - _, _ = self.call_component(implicit, {'time': timedelta(seconds=5)}) - assert implicit.times_called == 1, 'should not re-compute output' - _, _ = self.call_component(implicit, {'time': timedelta(seconds=10)}) - assert implicit.times_called == 2, 'should re-compute output' - _, _ = self.call_component(implicit, {'time': timedelta(seconds=15)}) - assert implicit.times_called == 2, 'should not re-compute output' - _, _ = self.call_component(implicit, {'time': timedelta(seconds=20)}) - assert implicit.times_called == 3, 'should re-compute output' - _, _ = self.call_component(implicit, {'time': timedelta(seconds=30)}) - assert implicit.times_called == 4, 'should re-compute output' - _, _ = self.call_component(implicit, {'time': timedelta(seconds=45)}) - assert implicit.times_called == 5, 'should re-compute output' - _, _ = self.call_component(implicit, {'time': timedelta(seconds=50)}) - assert implicit.times_called == 5, 'should not re-compute output' - - def test_update_interval_on_datetime(self): - input_properties = {} - diagnostic_properties = {} - output_properties = {} - diagnostic_output = {} - output_state = {} - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, output_state, - update_interval=timedelta(seconds=10) - ) - dt = datetime(2010, 1, 1) - _, _ = self.call_component( - implicit, {'time': dt + timedelta(seconds=0)}) - assert implicit.times_called == 1 - _, _ = self.call_component( - implicit, {'time': dt + timedelta(seconds=0)}) - assert implicit.times_called == 1, 'should not re-compute output' - _, _ = self.call_component( - implicit, {'time': dt + timedelta(seconds=5)}) - assert implicit.times_called == 1, 'should not re-compute output' - _, _ = self.call_component( - implicit, {'time': dt + timedelta(seconds=10)}) - assert implicit.times_called == 2, 'should re-compute output' - _, _ = self.call_component( - implicit, {'time': dt + timedelta(seconds=15)}) - assert implicit.times_called == 2, 'should not re-compute output' - _, _ = self.call_component( - implicit, {'time': dt + timedelta(seconds=20)}) - assert implicit.times_called == 3, 'should re-compute output' - _, _ = self.call_component( - implicit, {'time': dt + timedelta(seconds=30)}) - assert implicit.times_called == 4, 'should re-compute output' - _, _ = self.call_component( - implicit, {'time': dt + timedelta(seconds=45)}) - assert implicit.times_called == 5, 'should re-compute output' - _, _ = self.call_component( - implicit, {'time': dt + timedelta(seconds=50)}) - assert implicit.times_called == 5, 'should not re-compute output' - def test_tendencies_in_diagnostics_no_tendency(self): input_properties = {} diagnostic_properties = {} diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py new file mode 100644 index 0000000..5bec821 --- /dev/null +++ b/tests/test_wrapper.py @@ -0,0 +1,678 @@ +from datetime import timedelta, datetime +import unittest +from sympl import ( + Prognostic, Implicit, Diagnostic, UpdateFrequencyWrapper, ScalingWrapper, + TimeDifferencingWrapper, DataArray, ImplicitPrognostic +) +from test_base_components import ( + MockPrognostic, MockImplicit, MockImplicitPrognostic, MockDiagnostic) +import pytest +import numpy as np + + +class MockEmptyPrognostic(MockPrognostic): + + def __init__(self, **kwargs): + super(MockEmptyPrognostic, self).__init__( + input_properties={}, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + **kwargs + ) + + +class MockEmptyImplicitPrognostic(MockImplicitPrognostic): + def __init__(self, **kwargs): + super(MockEmptyImplicitPrognostic, self).__init__( + input_properties={}, + diagnostic_properties={}, + tendency_properties={}, + diagnostic_output={}, + tendency_output={}, + **kwargs + ) + + +class MockEmptyDiagnostic(MockDiagnostic): + + def __init__(self, **kwargs): + super(MockEmptyDiagnostic, self).__init__( + input_properties={}, + diagnostic_properties={}, + diagnostic_output={}, + **kwargs + ) + + +class MockEmptyImplicit(MockImplicit): + + def __init__(self, **kwargs): + super(MockEmptyImplicit, self).__init__( + input_properties={}, + diagnostic_properties={}, + output_properties={}, + diagnostic_output={}, + state_output={}, + **kwargs + ) + + +class UpdateFrequencyBase(object): + + def get_component(self): + raise NotImplementedError() + + def call_component(self, component, state): + raise NotImplementedError() + + def test_set_update_frequency_calls_initially(self): + component = UpdateFrequencyWrapper(self.get_component(), timedelta(hours=1)) + assert isinstance(component, self.component_type) + state = {'time': timedelta(hours=0)} + result = self.call_component(component, state) + assert component.times_called == 1 + + def test_set_update_frequency_does_not_repeat_call_at_same_timedelta(self): + component = UpdateFrequencyWrapper(self.get_component(), timedelta(hours=1)) + assert isinstance(component, self.component_type) + state = {'time': timedelta(hours=0)} + result = self.call_component(component, state) + result = self.call_component(component, state) + assert component.times_called == 1 + + def test_set_update_frequency_does_not_repeat_call_at_same_datetime(self): + component = UpdateFrequencyWrapper(self.get_component(), timedelta(hours=1)) + assert isinstance(component, self.component_type) + state = {'time': datetime(2010, 1, 1)} + result = self.call_component(component, state) + result = self.call_component(component, state) + assert component.times_called == 1 + + def test_set_update_frequency_updates_result_when_equal(self): + component = UpdateFrequencyWrapper(self.get_component(), timedelta(hours=1)) + assert isinstance(component, self.component_type) + result = self.call_component(component, {'time': timedelta(hours=0)}) + result = self.call_component(component, {'time': timedelta(hours=1)}) + assert component.times_called == 2 + + def test_set_update_frequency_updates_result_when_greater(self): + component = UpdateFrequencyWrapper(self.get_component(), timedelta(hours=1)) + assert isinstance(component, self.component_type) + result = self.call_component(component, {'time': timedelta(hours=0)}) + result = self.call_component(component, {'time': timedelta(hours=2)}) + assert component.times_called == 2 + + def test_set_update_frequency_does_not_update_when_less(self): + component = UpdateFrequencyWrapper(self.get_component(), timedelta(hours=1)) + assert isinstance(component, self.component_type) + result = self.call_component(component, {'time': timedelta(hours=0)}) + result = self.call_component(component, {'time': timedelta(minutes=59)}) + assert component.times_called == 1 + + +class PrognosticUpdateFrequencyTests(unittest.TestCase, UpdateFrequencyBase): + + component_type = Prognostic + + def get_component(self): + return MockEmptyPrognostic() + + def call_component(self, component, state): + return component(state) + + +class ImplicitPrognosticUpdateFrequencyTests(unittest.TestCase, UpdateFrequencyBase): + + component_type = ImplicitPrognostic + + def get_component(self): + return MockEmptyImplicitPrognostic() + + def call_component(self, component, state): + return component(state, timestep=timedelta(hours=1)) + + +class ImplicitUpdateFrequencyTests(unittest.TestCase, UpdateFrequencyBase): + + component_type = Implicit + + def get_component(self): + return MockEmptyImplicit() + + def call_component(self, component, state): + return component(state, timedelta(minutes=1)) + + +class DiagnosticUpdateFrequencyTests(unittest.TestCase, UpdateFrequencyBase): + + component_type = Diagnostic + + def get_component(self): + return MockEmptyDiagnostic() + + def call_component(self, component, state): + return component(state) + + +def test_scaled_component_wrong_type(): + class WrongObject(object): + def __init__(self): + self.a = 1 + + wrong_component = WrongObject() + + with pytest.raises(TypeError): + ScalingWrapper(wrong_component) + + +class ScalingInputMixin(object): + + def test_inputs_no_scaling(self): + self.input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + }, + } + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ), + } + base_component = self.get_component() + component = ScalingWrapper(base_component, input_scale_factors={}) + assert isinstance(component, self.component_type) + self.call_component(component, state) + assert base_component.state_given.keys() == state.keys() + assert np.all(base_component.state_given['input1'] == state['input1'].values) + + def test_inputs_one_scaling(self): + self.input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + }, + 'input2': { + 'dims': ['dim1'], + 'units': 'm', + }, + } + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ), + 'input2': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ), + } + base_component = self.get_component() + component = ScalingWrapper( + base_component, + input_scale_factors={ + 'input1': 10. + }) + assert isinstance(component, self.component_type) + self.call_component(component, state) + assert base_component.state_given.keys() == state.keys() + assert np.all(base_component.state_given['input1'] == state['input1'].values * 10.) + assert np.all(base_component.state_given['input2'] == state['input2'].values) + + def test_inputs_two_scalings(self): + self.input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + }, + 'input2': { + 'dims': ['dim1'], + 'units': 'm', + }, + } + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ), + 'input2': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ), + } + base_component = self.get_component() + component = ScalingWrapper( + base_component, + input_scale_factors={ + 'input1': 10., + 'input2': 5., + }) + assert isinstance(component, self.component_type) + self.call_component(component, state) + assert base_component.state_given.keys() == state.keys() + assert np.all(base_component.state_given['input1'] == 10.) + assert np.all(base_component.state_given['input2'] == 5.) + + def test_inputs_one_scaling_with_unit_conversion(self): + self.input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + }, + 'input2': { + 'dims': ['dim1'], + 'units': 'm', + }, + } + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'km'} + ), + 'input2': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ), + } + base_component = self.get_component() + component = ScalingWrapper( + base_component, + input_scale_factors={ + 'input1': 0.5 + }) + assert isinstance(component, self.component_type) + self.call_component(component, state) + assert base_component.state_given.keys() == state.keys() + assert np.all(base_component.state_given['input1'] == 500.) + assert np.all(base_component.state_given['input2'] == 1.) + + +class ScalingOutputMixin(object): + + def test_output_no_scaling(self): + self.output_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + self.output_state = { + 'diag1': np.ones([10]) + } + base_component = self.get_component() + component = ScalingWrapper( + base_component, + output_scale_factors={}, + ) + assert isinstance(component, self.component_type) + state = {'time': timedelta(0)} + outputs = self.get_outputs(self.call_component(component, state)) + assert outputs.keys() == self.output_state.keys() + assert np.all(outputs['diag1'] == 1.) + + def test_output_one_scaling(self): + self.output_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + self.output_state = { + 'diag1': np.ones([10]) + } + base_component = self.get_component() + component = ScalingWrapper( + base_component, + output_scale_factors={ + 'diag1': 10., + }, + ) + assert isinstance(component, self.component_type) + state = {'time': timedelta(0)} + outputs = self.get_outputs( + self.call_component(component, state)) + assert outputs.keys() == self.output_state.keys() + assert np.all(outputs['diag1'] == 10.) + + def test_output_no_scaling_when_input_scaled(self): + self.input_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + self.output_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + self.output_state = { + 'diag1': np.ones([10]) + } + base_component = self.get_component() + component = ScalingWrapper( + base_component, + input_scale_factors={ + 'diag1': 10., + }, + ) + assert isinstance(component, self.component_type) + state = { + 'time': timedelta(0), + 'diag1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + outputs = self.get_outputs( + self.call_component(component, state)) + assert outputs.keys() == self.output_state.keys() + assert np.all(outputs['diag1'] == 1.) + + +class ScalingDiagnosticMixin(object): + + def test_diagnostic_no_scaling(self): + self.diagnostic_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + self.diagnostic_output = { + 'diag1': np.ones([10]) + } + base_component = self.get_component() + component = ScalingWrapper( + base_component, + diagnostic_scale_factors={}, + ) + assert isinstance(component, self.component_type) + state = {'time': timedelta(0)} + diagnostics = self.get_diagnostics(self.call_component(component, state)) + assert diagnostics.keys() == self.diagnostic_output.keys() + assert np.all(diagnostics['diag1'] == 1.) + + def test_diagnostic_one_scaling(self): + self.diagnostic_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + self.diagnostic_output = { + 'diag1': np.ones([10]) + } + base_component = self.get_component() + component = ScalingWrapper( + base_component, + diagnostic_scale_factors={ + 'diag1': 10., + }, + ) + assert isinstance(component, self.component_type) + state = {'time': timedelta(0)} + diagnostics = self.get_diagnostics( + self.call_component(component, state)) + assert diagnostics.keys() == self.diagnostic_output.keys() + assert np.all(diagnostics['diag1'] == 10.) + + def test_diagnostic_no_scaling_when_input_scaled(self): + self.input_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + self.diagnostic_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + self.diagnostic_output = { + 'diag1': np.ones([10]) + } + base_component = self.get_component() + component = ScalingWrapper( + base_component, + input_scale_factors={ + 'diag1': 10., + }, + ) + assert isinstance(component, self.component_type) + state = { + 'time': timedelta(0), + 'diag1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + diagnostics = self.get_diagnostics( + self.call_component(component, state)) + assert diagnostics.keys() == self.diagnostic_output.keys() + assert np.all(diagnostics['diag1'] == 1.) + + +class ScalingTendencyMixin(object): + + def test_tendency_no_scaling(self): + self.tendency_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + self.tendency_output = { + 'diag1': np.ones([10]) + } + base_component = self.get_component() + component = ScalingWrapper( + base_component, + tendency_scale_factors={}, + ) + assert isinstance(component, self.component_type) + state = {'time': timedelta(0)} + tendencies = self.get_tendencies(self.call_component(component, state)) + assert tendencies.keys() == self.tendency_output.keys() + assert np.all(tendencies['diag1'] == 1.) + + def test_tendency_one_scaling(self): + self.tendency_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + self.tendency_output = { + 'diag1': np.ones([10]) + } + base_component = self.get_component() + component = ScalingWrapper( + base_component, + tendency_scale_factors={ + 'diag1': 10., + }, + ) + assert isinstance(component, self.component_type) + state = {'time': timedelta(0)} + tendencies = self.get_tendencies( + self.call_component(component, state)) + assert tendencies.keys() == self.tendency_output.keys() + assert np.all(tendencies['diag1'] == 10.) + + def test_tendency_no_scaling_when_input_scaled(self): + self.input_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + self.tendency_properties = { + 'diag1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + self.tendency_output = { + 'diag1': np.ones([10]) + } + base_component = self.get_component() + component = ScalingWrapper( + base_component, + input_scale_factors={ + 'diag1': 10., + }, + ) + assert isinstance(component, self.component_type) + state = { + 'time': timedelta(0), + 'diag1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + tendencies = self.get_tendencies( + self.call_component(component, state)) + assert tendencies.keys() == self.tendency_output.keys() + assert np.all(tendencies['diag1'] == 1.) + + +class DiagnosticScalingTests( + unittest.TestCase, ScalingInputMixin, ScalingDiagnosticMixin): + + component_type = Diagnostic + + def setUp(self): + self.input_properties = {} + self.diagnostic_properties = {} + self.diagnostic_output = {} + + def get_component(self): + return MockDiagnostic( + self.input_properties, + self.diagnostic_properties, + self.diagnostic_output + ) + + def get_diagnostics(self, output): + return output + + def call_component(self, component, state): + return component(state) + + +class PrognosticScalingTests( + unittest.TestCase, ScalingInputMixin, ScalingDiagnosticMixin, ScalingTendencyMixin): + + component_type = Prognostic + + def setUp(self): + self.input_properties = {} + self.diagnostic_properties = {} + self.tendency_properties = {} + self.diagnostic_output = {} + self.tendency_output = {} + + def get_component(self): + return MockPrognostic( + self.input_properties, + self.diagnostic_properties, + self.tendency_properties, + self.diagnostic_output, + self.tendency_output, + ) + + def get_diagnostics(self, output): + return output[1] + + def get_tendencies(self, output): + return output[0] + + def call_component(self, component, state): + return component(state) + + +class ImplicitPrognosticScalingTests( + unittest.TestCase, ScalingInputMixin, ScalingDiagnosticMixin, + ScalingTendencyMixin): + + component_type = ImplicitPrognostic + + def setUp(self): + self.input_properties = {} + self.diagnostic_properties = {} + self.tendency_properties = {} + self.diagnostic_output = {} + self.tendency_output = {} + + def get_component(self): + return MockImplicitPrognostic( + self.input_properties, + self.diagnostic_properties, + self.tendency_properties, + self.diagnostic_output, + self.tendency_output, + ) + + def get_diagnostics(self, output): + return output[1] + + def get_tendencies(self, output): + return output[0] + + def call_component(self, component, state): + return component(state, timedelta(hours=1)) + + +class ImplicitScalingTests( + unittest.TestCase, ScalingInputMixin, ScalingDiagnosticMixin, + ScalingOutputMixin): + + component_type = Implicit + + def setUp(self): + self.input_properties = {} + self.diagnostic_properties = {} + self.output_properties = {} + self.diagnostic_output = {} + self.output_state = {} + + def get_component(self): + return MockImplicit( + self.input_properties, + self.diagnostic_properties, + self.output_properties, + self.diagnostic_output, + self.output_state, + ) + + def get_diagnostics(self, output): + return output[0] + + def get_outputs(self, output): + return output[1] + + def call_component(self, component, state): + return component(state, timedelta(hours=1)) + +if __name__ == '__main__': + pytest.main([__file__]) From e0c7e4cc124ccf03c085d6d2689e95a49e18ed0e Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 4 Apr 2018 10:28:56 -0700 Subject: [PATCH 45/98] Python 3 fixes, added check that __init__ is called to base components * Now uses six to assign metaclass so it works in both 2 and 3, previously did not work in Python 3. * test_wrapper now defines its mocks directly to avoid import so the tests work in Python 3. * Component base classes now check to ensure that the __init__ method was called when their __call__ method is called, so that a more informative error message appears than would otherwise occur when an attribute is missing. --- HISTORY.rst | 5 + sympl/_core/base_components.py | 339 ++++++++++++++++++++------------- sympl/_core/wrappers.py | 35 +++- tests/test_base_components.py | 71 +++++++ tests/test_wrapper.py | 100 +++++++++- 5 files changed, 407 insertions(+), 143 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8d5b0ca..d75ab18 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -39,6 +39,7 @@ Latest Right now this only checks for the presence and lack of presence of component attributes, and correct signature of __call__. Later it may also check properties dictionaries for consistency, or perform other checks. +* Fixed a bug where ABCMeta was not being used in Python 3. Breaking changes ~~~~~~~~~~~~~~~~ @@ -76,6 +77,10 @@ Breaking changes * dims_like is obsolete as a result, and is no longer used. `dims` should be used instead. If present, `dims` from input properties will be used as default. +* Components will now raise an exception when __call__ of the component base + class (e.g. Implicit, Prognostic, etc.) if the __init__ method of the base + class has not been called, telling the user that the component __init__ + method should make a call to the superclass init. v0.3.2 ------ diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index da2c344..c919636 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -5,6 +5,7 @@ InvalidPropertyDictError, ComponentExtraOutputError, ComponentMissingOutputError, InvalidStateError) from inspect import getargspec +from six import add_metaclass def option_or_default(option, default): @@ -27,54 +28,64 @@ def is_component_base_class(cls): return cls in (Implicit, Prognostic, ImplicitPrognostic, Diagnostic) +def get_kwarg_defaults(func): + return_dict = {} + argspec = getargspec(func) + if argspec.defaults is not None: + n = len(argspec.args) - 1 + for i, default in enumerate(reversed(argspec.defaults)): + return_dict[argspec.args[n-i]] = default + return return_dict + + class ComponentMeta(abc.ABCMeta): - def __instancecheck__(self, instance): - if is_component_class(instance.__class__) or not is_component_base_class(self): - return issubclass(instance.__class__, self) - else: - check_attributes = ( - 'input_properties', - 'tendency_properties', - 'diagnostic_properties', - 'output_properties', - '__call__', - 'array_call', - 'tendencies_in_diagnostics', - 'name', - ) - required_attributes = list( - att for att in check_attributes if hasattr(self, att) - ) - disallowed_attributes = list( - att for att in check_attributes if att not in required_attributes - ) - if 'name' in disallowed_attributes: # name is always allowed - disallowed_attributes.remove('name') + def __instancecheck__(cls, instance): + if is_component_class(instance.__class__) or not is_component_base_class(cls): + return issubclass(instance.__class__, cls) + else: # checking if non-inheriting instance is a duck-type of a component base class + required_attributes, disallowed_attributes = cls.__get_attribute_requirements() has_attributes = ( all(hasattr(instance, att) for att in required_attributes) and not any(hasattr(instance, att) for att in disallowed_attributes) ) - if hasattr(self, '__call__') and not hasattr(instance, '__call__'): + if hasattr(cls, '__call__') and not hasattr(instance, '__call__'): return False - elif hasattr(self, '__call__'): - timestep_in_class_call = 'timestep' in getargspec(self.__call__).args + elif hasattr(cls, '__call__'): + timestep_in_class_call = 'timestep' in getargspec(cls.__call__).args instance_argspec = getargspec(instance.__call__) timestep_in_instance_call = 'timestep' in instance_argspec.args - instance_defaults = {} - if instance_argspec.defaults is not None: - n = len(instance_argspec.args) - 1 - for i, default in enumerate(reversed(instance_argspec.defaults)): - instance_defaults[instance_argspec.args[n-i]] = default - timestep_optional = ( + instance_defaults = get_kwarg_defaults(instance.__call__) + timestep_is_optional = ( 'timestep' in instance_defaults.keys() and instance_defaults['timestep'] is None) - has_correct_spec = (timestep_in_class_call == timestep_in_instance_call) or timestep_optional + has_correct_spec = (timestep_in_class_call == timestep_in_instance_call) or timestep_is_optional else: raise RuntimeError( 'Cannot check instance type on component subclass that has ' 'no __call__ method') return has_attributes and has_correct_spec + def __get_attribute_requirements(cls): + check_attributes = ( + 'input_properties', + 'tendency_properties', + 'diagnostic_properties', + 'output_properties', + '__call__', + 'array_call', + 'tendencies_in_diagnostics', + 'name', + ) + required_attributes = list( + att for att in check_attributes if hasattr(cls, att) + ) + disallowed_attributes = list( + att for att in check_attributes if att not in required_attributes + ) + if 'name' in disallowed_attributes: # name is always allowed + disallowed_attributes.remove('name') + return required_attributes, disallowed_attributes + def check_overlapping_aliases(properties, properties_name): defined_aliases = set() @@ -90,10 +101,11 @@ def check_overlapping_aliases(properties, properties_name): ) -class InputMixin(object): +class InputChecker(object): - def __init__(self): - for name, properties in self.input_properties.items(): + def __init__(self, component): + self.component = component + for name, properties in self.component.input_properties.items(): if 'units' not in properties.keys(): raise InvalidPropertyDictError( 'Input properties do not have units defined for {}'.format(name)) @@ -101,39 +113,40 @@ def __init__(self): raise InvalidPropertyDictError( 'Input properties do not have dims defined for {}'.format(name) ) - check_overlapping_aliases(self.input_properties, 'input') - super(InputMixin, self).__init__() + check_overlapping_aliases(self.component.input_properties, 'input') + super(InputChecker, self).__init__() - def _check_inputs(self, state): - for key in self.input_properties.keys(): + def check_inputs(self, state): + for key in self.component.input_properties.keys(): if key not in state.keys(): raise InvalidStateError('Missing input quantity {}'.format(key)) -class TendencyMixin(object): +class TendencyChecker(object): - def __init__(self): - for name, properties in self.tendency_properties.items(): + def __init__(self, component): + self.component = component + for name, properties in self.component.tendency_properties.items(): if 'units' not in properties.keys(): raise InvalidPropertyDictError( 'Tendency properties do not have units defined for {}'.format(name)) - if 'dims' not in properties.keys() and name not in self.input_properties.keys(): + if 'dims' not in properties.keys() and name not in self.component.input_properties.keys(): raise InvalidPropertyDictError( 'Tendency properties do not have dims defined for {}'.format(name) ) - check_overlapping_aliases(self.tendency_properties, 'tendency') - super(TendencyMixin, self).__init__() + check_overlapping_aliases(self.component.tendency_properties, 'tendency') + super(TendencyChecker, self).__init__() @property def _wanted_tendency_aliases(self): wanted_tendency_aliases = {} - for name, properties in self.tendency_properties.items(): + for name, properties in self.component.tendency_properties.items(): wanted_tendency_aliases[name] = [] if 'alias' in properties.keys(): wanted_tendency_aliases[name].append(properties['alias']) - if (name in self.input_properties.keys() and - 'alias' in self.input_properties[name].keys()): - wanted_tendency_aliases[name].append(self.input_properties[name]['alias']) + if (name in self.component.input_properties.keys() and + 'alias' in self.component.input_properties[name].keys()): + wanted_tendency_aliases[name].append(self.component.input_properties[name]['alias']) return wanted_tendency_aliases def _check_missing_tendencies(self, tendency_dict): @@ -159,47 +172,50 @@ def _check_extra_tendencies(self, tendency_dict): 'tendency_properties'.format( self.__class__.__name__, ', '.join(extra_tendencies))) - def _check_tendencies(self, tendency_dict): + def check_tendencies(self, tendency_dict): self._check_missing_tendencies(tendency_dict) self._check_extra_tendencies(tendency_dict) -class DiagnosticMixin(object): +class DiagnosticChecker(object): - def __init__(self): - for name, properties in self.diagnostic_properties.items(): + def __init__(self, component): + self.component = component + self._ignored_diagnostics = [] + for name, properties in component.diagnostic_properties.items(): if 'units' not in properties.keys(): raise InvalidPropertyDictError( 'Diagnostic properties do not have units defined for {}'.format(name)) - if 'dims' not in properties.keys() and name not in self.input_properties.keys(): + if 'dims' not in properties.keys() and name not in component.input_properties.keys(): raise InvalidPropertyDictError( 'Diagnostic properties do not have dims defined for {}'.format(name) ) - check_overlapping_aliases(self.diagnostic_properties, 'diagnostic') - super(DiagnosticMixin, self).__init__() + check_overlapping_aliases(component.diagnostic_properties, 'diagnostic') @property def _wanted_diagnostic_aliases(self): wanted_diagnostic_aliases = {} - for name, properties in self.diagnostic_properties.items(): + for name, properties in self.component.diagnostic_properties.items(): wanted_diagnostic_aliases[name] = [] if 'alias' in properties.keys(): wanted_diagnostic_aliases[name].append(properties['alias']) - if (name in self.input_properties.keys() and - 'alias' in self.input_properties[name].keys()): - wanted_diagnostic_aliases[name].append(self.input_properties[name]['alias']) + if (name in self.component.input_properties.keys() and + 'alias' in self.component.input_properties[name].keys()): + wanted_diagnostic_aliases[name].append( + self.component.input_properties[name]['alias']) return wanted_diagnostic_aliases def _check_missing_diagnostics(self, diagnostics_dict): missing_diagnostics = set() for name, aliases in self._wanted_diagnostic_aliases.items(): if (name not in diagnostics_dict.keys() and + name not in self._ignored_diagnostics and not any(alias in diagnostics_dict.keys() for alias in aliases)): missing_diagnostics.add(name) if len(missing_diagnostics) > 0: raise ComponentMissingOutputError( 'Component {} did not compute diagnostic(s) {}'.format( - self.__class__.__name__, ', '.join(missing_diagnostics))) + self.component.__class__.__name__, ', '.join(missing_diagnostics))) def _check_extra_diagnostics(self, diagnostics_dict): wanted_set = set() @@ -211,38 +227,42 @@ def _check_extra_diagnostics(self, diagnostics_dict): raise ComponentExtraOutputError( 'Component {} computed diagnostic(s) {} which are not in ' 'diagnostic_properties'.format( - self.__class__.__name__, ', '.join(extra_diagnostics))) + self.component.__class__.__name__, ', '.join(extra_diagnostics))) + + def set_ignored_diagnostics(self, ignored_diagnostics): + self._ignored_diagnostics = ignored_diagnostics - def _check_diagnostics(self, diagnostics_dict): + def check_diagnostics(self, diagnostics_dict): self._check_missing_diagnostics(diagnostics_dict) self._check_extra_diagnostics(diagnostics_dict) -class OutputMixin(object): +class OutputChecker(object): - def __init__(self): - for name, properties in self.output_properties.items(): + def __init__(self, component): + self.component = component + for name, properties in self.component.output_properties.items(): if 'units' not in properties.keys(): raise InvalidPropertyDictError( 'Output properties do not have units defined for {}'.format(name)) - if 'dims' not in properties.keys() and name not in self.input_properties.keys(): + if 'dims' not in properties.keys() and name not in self.component.input_properties.keys(): raise InvalidPropertyDictError( 'Output properties do not have dims defined for {}'.format(name) ) - check_overlapping_aliases(self.output_properties, 'output') - super(OutputMixin, self).__init__() + check_overlapping_aliases(self.component.output_properties, 'output') + super(OutputChecker, self).__init__() @property def _wanted_output_aliases(self): wanted_output_aliases = {} - for name, properties in self.output_properties.items(): + for name, properties in self.component.output_properties.items(): wanted_output_aliases[name] = [] if 'alias' in properties.keys(): wanted_output_aliases[name].append(properties['alias']) - if (name in self.input_properties.keys() and - 'alias' in self.input_properties[name].keys()): + if (name in self.component.input_properties.keys() and + 'alias' in self.component.input_properties[name].keys()): wanted_output_aliases[name].append( - self.input_properties[name]['alias']) + self.component.input_properties[name]['alias']) return wanted_output_aliases def _check_missing_outputs(self, outputs_dict): @@ -269,12 +289,13 @@ def _check_extra_outputs(self, outputs_dict): 'output_properties'.format( self.__class__.__name__, ', '.join(extra_outputs))) - def _check_outputs(self, output_dict): + def check_outputs(self, output_dict): self._check_missing_outputs(output_dict) self._check_extra_outputs(output_dict) -class Implicit(DiagnosticMixin, OutputMixin, InputMixin): +@add_metaclass(ComponentMeta) +class Implicit(object): """ Attributes ---------- @@ -306,7 +327,6 @@ class Implicit(DiagnosticMixin, OutputMixin, InputMixin): A label to be used for this object, for example as would be used for Y in the name "X_tendency_from_Y". """ - __metaclass__ = ComponentMeta time_unit_name = 's' time_unit_timedelta = timedelta(seconds=1) @@ -364,10 +384,14 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): """ self._tendencies_in_diagnostics = tendencies_in_diagnostics self.name = name or self.__class__.__name__.lower() + super(Implicit, self).__init__() + self._input_checker = InputChecker(self) + self._diagnostic_checker = DiagnosticChecker(self) + self._output_checker = OutputChecker(self) if tendencies_in_diagnostics: - self._added_tendency_properties = self._insert_tendency_properties() - else: - self._added_tendency_properties = set() + self._diagnostic_checker.set_ignored_diagnostics( + self._insert_tendency_properties()) + self.__initialized = True super(Implicit, self).__init__() def _insert_tendency_properties(self): @@ -412,18 +436,6 @@ def _insert_tendency_properties(self): added_names.append(tendency_name) return added_names - def _check_missing_diagnostics(self, diagnostics_dict): - missing_diagnostics = set() - for name, aliases in self._wanted_diagnostic_aliases.items(): - if (name not in diagnostics_dict.keys() and - name not in self._added_tendency_properties and - not any(alias in diagnostics_dict.keys() for alias in aliases)): - missing_diagnostics.add(name) - if len(missing_diagnostics) > 0: - raise ComponentMissingOutputError( - 'Component {} did not compute diagnostics {}'.format( - self.__class__.__name__, ', '.join(missing_diagnostics))) - def _get_tendency_name(self, name): return '{}_tendency_from_{}'.format(name, self.name) @@ -431,6 +443,20 @@ def _get_tendency_name(self, name): def tendencies_in_diagnostics(self): return self._tendencies_in_diagnostics # value cannot be modified + def __check_self_is_initialized(self): + try: + initialized = self.__initialized + except AttributeError: + initialized = False + if not initialized: + raise RuntimeError( + 'Component has not called __init__ of base class, likely ' + 'because its class {} is missing a call to ' + 'super({}, self).__init__(**kwargs) in its __init__ ' + 'method.'.format( + self.__class__.__name__, self.__class__.__name__) + ) + def __call__(self, state, timestep): """ Gets diagnostics from the current model state and steps the state @@ -460,12 +486,13 @@ def __call__(self, state, timestep): If state is not a valid input for the Implicit instance for other reasons. """ - self._check_inputs(state) + self.__check_self_is_initialized() + self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) raw_state['time'] = state['time'] raw_diagnostics, raw_new_state = self.array_call(raw_state, timestep) - self._check_diagnostics(raw_diagnostics) - self._check_outputs(raw_new_state) + self._diagnostic_checker.check_diagnostics(raw_diagnostics) + self._output_checker.check_outputs(raw_new_state) if self.tendencies_in_diagnostics: self._insert_tendencies_to_diagnostics( raw_state, raw_new_state, timestep, raw_diagnostics) @@ -512,7 +539,8 @@ def array_call(self, state, timestep): pass -class Prognostic(DiagnosticMixin, TendencyMixin, InputMixin): +@add_metaclass(ComponentMeta) +class Prognostic(object): """ Attributes ---------- @@ -536,7 +564,6 @@ class Prognostic(DiagnosticMixin, TendencyMixin, InputMixin): A label to be used for this object, for example as would be used for Y in the name "X_tendency_from_Y". """ - __metaclass__ = ComponentMeta @abc.abstractproperty def input_properties(self): @@ -592,9 +619,15 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): """ self._tendencies_in_diagnostics = tendencies_in_diagnostics self.name = name or self.__class__.__name__ - self._added_diagnostic_names = [] + self._input_checker = InputChecker(self) + self._tendency_checker = TendencyChecker(self) + self._diagnostic_checker = DiagnosticChecker(self) if self.tendencies_in_diagnostics: - self._insert_tendency_properties() + self._added_diagnostic_names = self._insert_tendency_properties() + self._diagnostic_checker.set_ignored_diagnostics(self._added_diagnostic_names) + else: + self._added_diagnostic_names = [] + self.__initialized = True super(Prognostic, self).__init__() @property @@ -602,6 +635,7 @@ def tendencies_in_diagnostics(self): return self._tendencies_in_diagnostics def _insert_tendency_properties(self): + added_names = [] for name, properties in self.tendency_properties.items(): tendency_name = self._get_tendency_name(name) if 'dims' in properties.keys(): @@ -612,11 +646,26 @@ def _insert_tendency_properties(self): 'units': properties['units'], 'dims': dims, } - self._added_diagnostic_names.append(tendency_name) + added_names.append(tendency_name) + return added_names def _get_tendency_name(self, name): return '{}_tendency_from_{}'.format(name, self.name) + def __check_self_is_initialized(self): + try: + initialized = self.__initialized + except AttributeError: + initialized = False + if not initialized: + raise RuntimeError( + 'Component has not called __init__ of base class, likely ' + 'because its class {} is missing a call to ' + 'super({}, self).__init__(**kwargs) in its __init__ ' + 'method.'.format( + self.__class__.__name__, self.__class__.__name__) + ) + def __call__(self, state): """ Gets tendencies and diagnostics from the passed model state. @@ -645,11 +694,13 @@ def __call__(self, state): InvalidStateError If state is not a valid input for the Prognostic instance. """ + self.__check_self_is_initialized() + self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) raw_state['time'] = state['time'] raw_tendencies, raw_diagnostics = self.array_call(raw_state) - self._check_tendencies(raw_tendencies) - self._check_diagnostics(raw_diagnostics) + self._tendency_checker.check_tendencies(raw_tendencies) + self._diagnostic_checker.check_diagnostics(raw_diagnostics) tendencies = restore_data_arrays_with_properties( raw_tendencies, self.tendency_properties, state, self.input_properties) @@ -666,18 +717,6 @@ def _insert_tendencies_to_diagnostics(self, tendencies, diagnostics): tendency_name = self._get_tendency_name(name) diagnostics[tendency_name] = value - def _check_missing_diagnostics(self, diagnostics_dict): - missing_diagnostics = set() - for name, aliases in self._wanted_diagnostic_aliases.items(): - if (name not in diagnostics_dict.keys() and - name not in self._added_diagnostic_names and - not any(alias in diagnostics_dict.keys() for alias in aliases)): - missing_diagnostics.add(name) - if len(missing_diagnostics) > 0: - raise ComponentMissingOutputError( - 'Component {} did not compute diagnostics {}'.format( - self.__class__.__name__, ', '.join(missing_diagnostics))) - @abc.abstractmethod def array_call(self, state): """ @@ -706,7 +745,8 @@ def array_call(self, state): pass -class ImplicitPrognostic(DiagnosticMixin, TendencyMixin, InputMixin): +@add_metaclass(ComponentMeta) +class ImplicitPrognostic(object): """ Attributes ---------- @@ -730,7 +770,6 @@ class ImplicitPrognostic(DiagnosticMixin, TendencyMixin, InputMixin): A label to be used for this object, for example as would be used for Y in the name "X_tendency_from_Y". """ - __metaclass__ = ComponentMeta @abc.abstractproperty def input_properties(self): @@ -785,8 +824,13 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): self._tendencies_in_diagnostics = tendencies_in_diagnostics self.name = name or self.__class__.__name__ self._added_diagnostic_names = [] + self._input_checker = InputChecker(self) + self._diagnostic_checker = DiagnosticChecker(self) + self._tendency_checker = TendencyChecker(self) if self.tendencies_in_diagnostics: - self._insert_tendency_properties() + self._added_diagnostic_names = self._insert_tendency_properties() + self._diagnostic_checker.set_ignored_diagnostics(self._added_diagnostic_names) + self.__initialized = True super(ImplicitPrognostic, self).__init__() @property @@ -794,6 +838,7 @@ def tendencies_in_diagnostics(self): return self._tendencies_in_diagnostics def _insert_tendency_properties(self): + added_names = [] for name, properties in self.tendency_properties.items(): tendency_name = self._get_tendency_name(name) if 'dims' in properties.keys(): @@ -804,11 +849,26 @@ def _insert_tendency_properties(self): 'units': properties['units'], 'dims': dims, } - self._added_diagnostic_names.append(tendency_name) + added_names.append(tendency_name) + return added_names def _get_tendency_name(self, name): return '{}_tendency_from_{}'.format(name, self.name) + def __check_self_is_initialized(self): + try: + initialized = self.__initialized + except AttributeError: + initialized = False + if not initialized: + raise RuntimeError( + 'Component has not called __init__ of base class, likely ' + 'because its class {} is missing a call to ' + 'super({}, self).__init__(**kwargs) in its __init__ ' + 'method.'.format( + self.__class__.__name__, self.__class__.__name__) + ) + def __call__(self, state, timestep): """ Gets tendencies and diagnostics from the passed model state. @@ -839,11 +899,13 @@ def __call__(self, state, timestep): InvalidStateError If state is not a valid input for the Prognostic instance. """ + self.__check_self_is_initialized() + self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) raw_state['time'] = state['time'] raw_tendencies, raw_diagnostics = self.array_call(raw_state, timestep) - self._check_tendencies(raw_tendencies) - self._check_diagnostics(raw_diagnostics) + self._tendency_checker.check_tendencies(raw_tendencies) + self._diagnostic_checker.check_diagnostics(raw_diagnostics) tendencies = restore_data_arrays_with_properties( raw_tendencies, self.tendency_properties, state, self.input_properties) @@ -861,18 +923,6 @@ def _insert_tendencies_to_diagnostics(self, tendencies, diagnostics): tendency_name = self._get_tendency_name(name) diagnostics[tendency_name] = value - def _check_missing_diagnostics(self, diagnostics_dict): - missing_diagnostics = set() - for name, aliases in self._wanted_diagnostic_aliases.items(): - if (name not in diagnostics_dict.keys() and - name not in self._added_diagnostic_names and - not any(alias in diagnostics_dict.keys() for alias in aliases)): - missing_diagnostics.add(name) - if len(missing_diagnostics) > 0: - raise ComponentMissingOutputError( - 'Component {} did not compute diagnostics {}'.format( - self.__class__.__name__, ', '.join(missing_diagnostics))) - @abc.abstractmethod def array_call(self, state, timestep): """ @@ -902,7 +952,8 @@ def array_call(self, state, timestep): """ -class Diagnostic(DiagnosticMixin, InputMixin): +@add_metaclass(ComponentMeta) +class Diagnostic(object): """ Attributes ---------- @@ -915,7 +966,6 @@ class Diagnostic(DiagnosticMixin, InputMixin): object is called, and values are dictionaries which indicate 'dims' and 'units'. """ - __metaclass__ = ComponentMeta @abc.abstractproperty def input_properties(self): @@ -950,8 +1000,25 @@ def __init__(self): """ Initializes the Implicit object. """ + self._input_checker = InputChecker(self) + self._diagnostic_checker = DiagnosticChecker(self) + self.__initialized = True super(Diagnostic, self).__init__() + def __check_self_is_initialized(self): + try: + initialized = self.__initialized + except AttributeError: + initialized = False + if not initialized: + raise RuntimeError( + 'Component has not called __init__ of base class, likely ' + 'because its class {} is missing a call to ' + 'super({}, self).__init__(**kwargs) in its __init__ ' + 'method.'.format( + self.__class__.__name__, self.__class__.__name__) + ) + def __call__(self, state): """ Gets diagnostics from the passed model state. @@ -975,10 +1042,12 @@ def __call__(self, state): InvalidStateError If state is not a valid input for the Prognostic instance. """ + self.__check_self_is_initialized() + self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) raw_state['time'] = state['time'] raw_diagnostics = self.array_call(raw_state) - self._check_diagnostics(raw_diagnostics) + self._diagnostic_checker.check_diagnostics(raw_diagnostics) diagnostics = restore_data_arrays_with_properties( raw_diagnostics, self.diagnostic_properties, state, self.input_properties) @@ -1005,8 +1074,8 @@ def array_call(self, state): """ +@add_metaclass(abc.ABCMeta) class Monitor(object): - __metaclass__ = abc.ABCMeta def __str__(self): return 'instance of {}(Monitor)'.format(self.__class__) diff --git a/sympl/_core/wrappers.py b/sympl/_core/wrappers.py index 6dcea27..f7104e1 100644 --- a/sympl/_core/wrappers.py +++ b/sympl/_core/wrappers.py @@ -7,6 +7,7 @@ class ScalingWrapper(object): """ Wraps any component and scales either inputs, outputs or tendencies by a floating point value. + Example ------- This is how the ScalingWrapper can be used to wrap a Prognostic. @@ -26,9 +27,10 @@ def __init__(self, diagnostic_scale_factors=None): """ Initializes the ScaledInputOutputWrapper object. + Args ---- - component : Prognostic, Implicit + component : Prognostic, Implicit, Diagnostic, ImplicitPrognostic The component to be wrapped. input_scale_factors : dict a dictionary whose keys are the inputs that will be scaled @@ -42,10 +44,12 @@ def __init__(self, diagnostic_scale_factors : dict a dictionary whose keys are the diagnostics that will be scaled and values are floating point scaling factors. + Returns ------- scaled_component : ScaledInputOutputWrapper the scaled version of the component + Raises ------ TypeError @@ -187,10 +191,10 @@ def __call__(self, state, timestep=None): class UpdateFrequencyWrapper(object): """ - Wraps a prognostic object so that when it is called, it only computes new + Wraps a component object so that when it is called, it only computes new output if sufficient time has passed, and otherwise returns its last - computed output. The Delayed object requires that the 'time' attribute is - set in the state, in addition to any requirements of the Prognostic + computed output. + Example ------- This how the wrapper should be used on a fictional Prognostic class @@ -202,10 +206,11 @@ class UpdateFrequencyWrapper(object): def __init__(self, component, update_timedelta): """ Initialize the UpdateFrequencyWrapper object. + Args ---- - component - The component object to be wrapped. + component : Prognostic, Implicit, Diagnostic, ImplicitPrognostic + The component to be wrapped. update_timedelta : timedelta The amount that state['time'] must differ from when output was cached before new output is computed. @@ -216,6 +221,24 @@ def __init__(self, component, update_timedelta): self._last_update_time = None def __call__(self, state, timestep=None, **kwargs): + """ + Call the underlying component, or return cached values instead if + insufficient time has passed since the last time cached values were + stored. + + Parameters + ---------- + state : dict + A model state dictionary. + timestep : timedelta, optional + A time step. If the underlying component does not use a timestep, + this will be discarded. If it does, this argument is required. + + Returns + ------- + *args + The return values of the underlying component. + """ if ((self._last_update_time is None) or (state['time'] >= self._last_update_time + self._update_timedelta)): diff --git a/tests/test_base_components.py b/tests/test_base_components.py index 34aac99..1f25dcf 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -116,6 +116,57 @@ class MockMonitor(Monitor): def store(self, state): return +class BadMockPrognostic(Prognostic): + + input_properties = {} + tendency_properties = {} + diagnostic_properties = {} + + def __init__(self): + pass + + def array_call(self, state): + return {}, {} + + +class BadMockImplicitPrognostic(ImplicitPrognostic): + + input_properties = {} + tendency_properties = {} + diagnostic_properties = {} + + def __init__(self): + pass + + def array_call(self, state, timestep): + return {}, {} + + +class BadMockDiagnostic(Diagnostic): + + input_properties = {} + diagnostic_properties = {} + + def __init__(self): + pass + + def array_call(self, state): + return {} + + +class BadMockImplicit(Implicit): + + input_properties = {} + diagnostic_properties = {} + output_properties = {} + + def __init__(self): + pass + + def array_call(self, state, timestep): + return {}, {} + + class PrognosticTests(unittest.TestCase): @@ -124,6 +175,11 @@ class PrognosticTests(unittest.TestCase): def call_component(self, component, state): return component(state) + def test_cannot_use_bad_component(self): + component = BadMockPrognostic() + with self.assertRaises(RuntimeError): + self.call_component(component, {'time': timedelta(0)}) + def test_subclass_check(self): class MyPrognostic(object): input_properties = {} @@ -1066,6 +1122,11 @@ class ImplicitPrognosticTests(PrognosticTests): def call_component(self, component, state): return component(state, timedelta(seconds=1)) + def test_cannot_use_bad_component(self): + component = BadMockImplicitPrognostic() + with self.assertRaises(RuntimeError): + self.call_component(component, {'time': timedelta(0)}) + def test_subclass_check(self): class MyImplicitPrognostic(object): input_properties = {} @@ -1177,6 +1238,11 @@ class DiagnosticTests(unittest.TestCase): def call_component(self, component, state): return component(state) + def test_cannot_use_bad_component(self): + component = BadMockDiagnostic() + with self.assertRaises(RuntimeError): + self.call_component(component, {'time': timedelta(0)}) + def test_subclass_check(self): class MyDiagnostic(object): input_properties = {} @@ -1603,6 +1669,11 @@ class ImplicitTests(unittest.TestCase): def call_component(self, component, state): return component(state, timedelta(seconds=1)) + def test_cannot_use_bad_component(self): + component = BadMockImplicit() + with self.assertRaises(RuntimeError): + self.call_component(component, {'time': timedelta(0)}) + def test_subclass_check(self): class MyImplicit(object): input_properties = {} diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index 5bec821..200c021 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -4,12 +4,108 @@ Prognostic, Implicit, Diagnostic, UpdateFrequencyWrapper, ScalingWrapper, TimeDifferencingWrapper, DataArray, ImplicitPrognostic ) -from test_base_components import ( - MockPrognostic, MockImplicit, MockImplicitPrognostic, MockDiagnostic) import pytest import numpy as np +class MockPrognostic(Prognostic): + + input_properties = None + diagnostic_properties = None + tendency_properties = None + + def __init__( + self, input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, **kwargs): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self.tendency_properties = tendency_properties + self.diagnostic_output = diagnostic_output + self.tendency_output = tendency_output + self.times_called = 0 + self.state_given = None + super(MockPrognostic, self).__init__(**kwargs) + + def array_call(self, state): + self.times_called += 1 + self.state_given = state + return self.tendency_output, self.diagnostic_output + + +class MockImplicitPrognostic(ImplicitPrognostic): + + input_properties = None + diagnostic_properties = None + tendency_properties = None + + def __init__( + self, input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, **kwargs): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self.tendency_properties = tendency_properties + self.diagnostic_output = diagnostic_output + self.tendency_output = tendency_output + self.times_called = 0 + self.state_given = None + self.timestep_given = None + super(MockImplicitPrognostic, self).__init__(**kwargs) + + def array_call(self, state, timestep): + self.times_called += 1 + self.state_given = state + self.timestep_given = timestep + return self.tendency_output, self.diagnostic_output + + +class MockDiagnostic(Diagnostic): + + input_properties = None + diagnostic_properties = None + + def __init__( + self, input_properties, diagnostic_properties, diagnostic_output, + **kwargs): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self.diagnostic_output = diagnostic_output + self.times_called = 0 + self.state_given = None + super(MockDiagnostic, self).__init__(**kwargs) + + def array_call(self, state): + self.times_called += 1 + self.state_given = state + return self.diagnostic_output + + +class MockImplicit(Implicit): + + input_properties = None + diagnostic_properties = None + output_properties = None + + def __init__( + self, input_properties, diagnostic_properties, output_properties, + diagnostic_output, state_output, + **kwargs): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self.output_properties = output_properties + self.diagnostic_output = diagnostic_output + self.state_output = state_output + self.times_called = 0 + self.state_given = None + self.timestep_given = None + super(MockImplicit, self).__init__(**kwargs) + + def array_call(self, state, timestep): + self.times_called += 1 + self.state_given = state + self.timestep_given = timestep + return self.diagnostic_output, self.state_output + + class MockEmptyPrognostic(MockPrognostic): def __init__(self, **kwargs): From f85dc8f530d5d84f3d9227fc7fa58c212a95b32e Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 4 Apr 2018 10:35:42 -0700 Subject: [PATCH 46/98] Added conditional import of getfullargspec over deprecated getargspec --- sympl/_core/base_components.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index c919636..3e4d629 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -4,8 +4,11 @@ from .exceptions import ( InvalidPropertyDictError, ComponentExtraOutputError, ComponentMissingOutputError, InvalidStateError) -from inspect import getargspec from six import add_metaclass +try: + from inspect import getfullargspec as getargspec +except ImportError: + from inspect import getargspec def option_or_default(option, default): From 866ea235fa949397b089c3ba847ede69be6fdd6c Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 9 Apr 2018 11:14:31 -0700 Subject: [PATCH 47/98] Added initialize_numpy_arrays_with_properties --- HISTORY.rst | 2 + sympl/__init__.py | 4 +- sympl/_core/state.py | 231 +++- tests/test_base_components.py | 1902 +++++++++++---------------------- tests/test_state.py | 313 ++++++ 5 files changed, 1143 insertions(+), 1309 deletions(-) create mode 100644 tests/test_state.py diff --git a/HISTORY.rst b/HISTORY.rst index d75ab18..04c4d2b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -40,6 +40,8 @@ Latest component attributes, and correct signature of __call__. Later it may also check properties dictionaries for consistency, or perform other checks. * Fixed a bug where ABCMeta was not being used in Python 3. +* Added initialize_numpy_arrays_with_properties which creates zero arrays for an output + properties dictionary. Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/__init__.py b/sympl/__init__.py index 5a7ebfe..7c7cc7f 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -21,7 +21,8 @@ get_component_aliases) from ._core.state import ( get_numpy_arrays_with_properties, - restore_data_arrays_with_properties) + restore_data_arrays_with_properties, + initialize_numpy_arrays_with_properties) from ._components import ( PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, @@ -44,6 +45,7 @@ get_numpy_array, jit, restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, + initialize_numpy_arrays_with_properties, get_component_aliases, PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, diff --git a/sympl/_core/state.py b/sympl/_core/state.py index d10fab6..71ab05a 100644 --- a/sympl/_core/state.py +++ b/sympl/_core/state.py @@ -54,7 +54,10 @@ def get_wildcard_matches_and_dim_lengths(state, property_dictionary): quantity_name, new_wildcard_names)) wildcard_names.extend( [name for name in new_wildcard_names if name not in wildcard_names]) - wildcard_names = tuple(wildcard_names) + if not any('dims' in p.keys() and '*' in p['dims'] for p in property_dictionary.values()): + wildcard_names = None # can't determine wildcard matches if there is no wildcard + else: + wildcard_names = tuple(wildcard_names) return wildcard_names, dim_lengths @@ -142,22 +145,72 @@ def get_numpy_array(data_array, out_dims, dim_lengths): return out_array -def restore_data_arrays_with_properties( - raw_arrays, output_properties, input_state, input_properties, - ignore_names=None): - if ignore_names is None: - ignore_names = [] +def initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties, dtype=np.float64): + """ + Parameters + ---------- + output_properties : dict + A dictionary whose keys are quantity names and values are dictionaries + with properties for those quantities. The property "dims" must be + present for each quantity not also present in input_properties. + input_state : dict + A state dictionary that was used as input to a component for which + DataArrays are being restored. + input_properties : dict + A dictionary whose keys are quantity names and values are dictionaries + with input properties for those quantities. The property "dims" must be + present, indicating the dimensions that the quantity was transformed to + when taken as input to a component. + + Returns + ------- + out_dict : dict + A dictionary whose keys are quantities and values are numpy arrays + corresponding to those quantities, with shapes determined from the + inputs to this function. + + Raises + ------ + InvalidPropertyDictError + When an output property is specified to have dims_like an input + property, but the arrays for the two properties have incompatible + shapes. + """ wildcard_names, dim_lengths = get_wildcard_matches_and_dim_lengths( input_state, input_properties) - for name, value in raw_arrays.items(): + dims_from_out_properties = extract_output_dims_properties( + output_properties, input_properties, []) + out_dict = {} + for name, out_dims in dims_from_out_properties.items(): + if '*' in out_dims and wildcard_names is not None: + _, out_shape = fill_dims_wildcard( + out_dims, dim_lengths, wildcard_names, expand_wildcard=False) + elif '*' in out_dims and wildcard_names is None: + raise InvalidPropertyDictError( + 'Cannot determine wildcard dimensions required for output if ' + 'there are no wildcard dimensions in input_properties') + else: + out_shape = [] + for dim in out_dims: + out_shape.append(dim_lengths[dim]) + out_dict[name] = np.zeros(out_shape, dtype=dtype) + return out_dict + + +def ensure_values_are_arrays(array_dict): + for name, value in array_dict.items(): if not isinstance(value, np.ndarray): - raw_arrays[name] = np.asarray(value) - out_dims_property = {} + array_dict[name] = np.asarray(value) + + +def extract_output_dims_properties(output_properties, input_properties, ignore_names): + return_array = {} for name, properties in output_properties.items(): if name in ignore_names: continue elif 'dims' in properties.keys(): - out_dims_property[name] = properties['dims'] + return_array[name] = properties['dims'] elif name not in input_properties.keys(): raise InvalidPropertyDictError( 'Output dims must be specified for {} in properties'.format(name)) @@ -165,54 +218,130 @@ def restore_data_arrays_with_properties( raise InvalidPropertyDictError( 'Input dims must be specified for {} in properties'.format(name)) else: - out_dims_property[name] = input_properties[name]['dims'] + return_array[name] = input_properties[name]['dims'] + return return_array + + +def fill_dims_wildcard( + out_dims, dim_lengths, wildcard_names, expand_wildcard=True): + i_wildcard = out_dims.index('*') + target_shape = [] + out_dims_without_wildcard = [] + for i, out_dim in enumerate(out_dims): + if i == i_wildcard and expand_wildcard: + target_shape.extend([dim_lengths[n] for n in wildcard_names]) + out_dims_without_wildcard.extend(wildcard_names) + elif i == i_wildcard and not expand_wildcard: + target_shape.append(np.product([dim_lengths[n] for n in wildcard_names])) + else: + target_shape.append(dim_lengths[out_dim]) + out_dims_without_wildcard.append(out_dim) + return out_dims_without_wildcard, target_shape + + +def expand_array_wildcard_dims(raw_array, target_shape, name, out_dims): + try: + out_array = np.reshape(raw_array, target_shape) + except ValueError: + raise InvalidPropertyDictError( + 'Failed to restore shape for output {} with raw shape {} ' + 'and target shape {}, are the output dims {} correct?'.format( + name, raw_array.shape, target_shape, + out_dims)) + return out_array + + +def get_alias_or_name(name, output_properties, input_properties): + if 'alias' in output_properties[name].keys(): + raw_name = output_properties[name]['alias'] + elif name in input_properties.keys() and 'alias' in input_properties[name].keys(): + raw_name = input_properties[name]['alias'] + else: + raw_name = name + return raw_name + + +def check_array_shape(out_dims, raw_array, name, dim_lengths): + if len(out_dims) != len(raw_array.shape): + raise InvalidPropertyDictError( + 'Returned array for {} has shape {} ' + 'which is incompatible with dims {} in properties'.format( + name, raw_array.shape, out_dims)) + for dim, length in zip(out_dims, raw_array.shape): + if dim in dim_lengths.keys() and dim_lengths[dim] != length: + raise InvalidPropertyDictError( + 'Dimension {} of quantity {} has length {}, but ' + 'another quantity has length {}'.format( + dim, name, length, dim_lengths[dim]) + ) + + +def restore_data_arrays_with_properties( + raw_arrays, output_properties, input_state, input_properties, + ignore_names=None): + """ + Parameters + ---------- + raw_arrays : dict + A dictionary whose keys are quantity names and values are numpy arrays + containing the data for those quantities. + output_properties : dict + A dictionary whose keys are quantity names and values are dictionaries + with properties for those quantities. The property "dims" must be + present for each quantity not also present in input_properties. All + other properties are included as attributes on the output DataArray + for that quantity, including "units" which is required. + input_state : dict + A state dictionary that was used as input to a component for which + DataArrays are being restored. + input_properties : dict + A dictionary whose keys are quantity names and values are dictionaries + with input properties for those quantities. The property "dims" must be + present, indicating the dimensions that the quantity was transformed to + when taken as input to a component. + ignore_names : iterable of str + Names to ignore when encountered in output_properties, will not be + included in the returned dictionary. + + Returns + ------- + out_dict : dict + A dictionary whose keys are quantities and values are DataArrays + corresponding to those quantities, with data, shapes and attributes + determined from the inputs to this function. + + Raises + ------ + InvalidPropertyDictError + When an output property is specified to have dims_like an input + property, but the arrays for the two properties have incompatible + shapes. + """ + raw_arrays = raw_arrays.copy() + if ignore_names is None: + ignore_names = [] + wildcard_names, dim_lengths = get_wildcard_matches_and_dim_lengths( + input_state, input_properties) + ensure_values_are_arrays(raw_arrays) + dims_from_out_properties = extract_output_dims_properties( + output_properties, input_properties, ignore_names) out_dict = {} - for name, dims in out_dims_property.items(): + for name, out_dims in dims_from_out_properties.items(): if name in ignore_names: continue - if 'alias' in output_properties[name].keys(): - raw_name = output_properties[name]['alias'] - elif name in input_properties.keys() and 'alias' in input_properties[name].keys(): - raw_name = input_properties[name]['alias'] - else: - raw_name = name - if '*' in dims: - i_wildcard = dims.index('*') - target_shape = [] - out_dims = [] - for i, length in enumerate(raw_arrays[raw_name].shape): - if i == i_wildcard: - target_shape.extend([dim_lengths[n] for n in wildcard_names]) - out_dims.extend(wildcard_names) - else: - target_shape.append(length) - out_dims.append(dims[i]) - try: - out_array = np.reshape(raw_arrays[raw_name], target_shape) - except ValueError: - raise InvalidPropertyDictError( - 'Failed to restore shape for output {} with raw shape {} ' - 'and target shape {}, are the output dims {} correct?'.format( - name, raw_arrays[raw_name].shape, target_shape, - out_dims_property[name])) + raw_name = get_alias_or_name(name, output_properties, input_properties) + if '*' in out_dims: + out_dims_without_wildcard, target_shape = fill_dims_wildcard( + out_dims, dim_lengths, wildcard_names) + out_array = expand_array_wildcard_dims( + raw_arrays[raw_name], target_shape, name, out_dims) else: - if len(dims) != len(raw_arrays[raw_name].shape): - raise InvalidPropertyDictError( - 'Returned array for {} has shape {} ' - 'which is incompatible with dims {} in properties'.format( - name, raw_arrays[raw_name].shape, dims)) - for dim, length in zip(dims, raw_arrays[raw_name].shape): - if dim in dim_lengths.keys() and dim_lengths[dim] != length: - raise InvalidPropertyDictError( - 'Dimension {} of quantity {} has length {}, but ' - 'another quantity has length {}'.format( - dim, name, length, dim_lengths[dim]) - ) - out_dims = dims + check_array_shape(out_dims, raw_arrays[raw_name], name, dim_lengths) + out_dims_without_wildcard = out_dims out_array = raw_arrays[raw_name] out_dict[name] = DataArray( out_array, - dims=out_dims, + dims=out_dims_without_wildcard, attrs={'units': output_properties[name]['units']} ) return out_dict diff --git a/tests/test_base_components.py b/tests/test_base_components.py index 1f25dcf..5a628a7 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -6,6 +6,7 @@ Prognostic, Diagnostic, Monitor, Implicit, ImplicitPrognostic, datetime, timedelta, DataArray, InvalidPropertyDictError, ComponentMissingOutputError, ComponentExtraOutputError, + InvalidStateError ) def same_list(list1, list2): @@ -167,14 +168,546 @@ def array_call(self, state, timestep): return {}, {} +class InputTestBase(): + + def test_cannot_overlap_input_aliases(self): + input_properties = { + 'input1': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'}, + 'input2': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'} + } + with self.assertRaises(InvalidPropertyDictError): + self.get_component(input_properties=input_properties) + + def test_raises_when_input_missing(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + component = self.get_component(input_properties=input_properties) + state = {'time': timedelta(0)} + with self.assertRaises(InvalidStateError): + self.call_component(component, state) + + def test_raises_when_input_incorrect_units(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + component = self.get_component(input_properties=input_properties) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.zeros([10]), + dims=['dim1'], + attrs={'units': 's'}, + ), + } + with self.assertRaises(InvalidStateError): + self.call_component(component, state) + + def test_raises_when_input_incorrect_dims(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + component = self.get_component(input_properties=input_properties) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.zeros([10]), + dims=['dim2'], + attrs={'units': 'm'}, + ), + } + with self.assertRaises(InvalidStateError): + self.call_component(component, state) + + def test_raises_when_input_conflicting_dim_lengths(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + }, + 'input1': { + 'dims': ['dim2'], + 'units': 'm', + } + } + component = self.get_component(input_properties=input_properties) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.zeros([10]), + dims=['dim1'], + attrs={'units': 'm'}, + ), + 'input2': DataArray( + np.zeros([7]), + dims=['dim1'], + attrs={'units': 'm'}, + ), + } + with self.assertRaises(InvalidStateError): + self.call_component(component, state) + + def test_collects_independent_wildcard_dims(self): + input_properties = { + 'input1': { + 'dims': ['*'], + 'units': 'm', + }, + 'input2': { + 'dims': ['*'], + 'units': 'm', + } + } + component = self.get_component(input_properties=input_properties) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.zeros([4]), + dims=['dim1'], + attrs={'units': 'm'}, + ), + 'input2': DataArray( + np.zeros([3]), + dims=['dim2'], + attrs={'units': 'm'}, + ), + } + self.call_component(component, state) + given = component.state_given + assert len(given.keys()) == 3 + assert 'time' in given.keys() + assert 'input1' in given.keys() + assert given['input1'].shape == (12,) + assert 'input2' in given.keys() + assert given['input2'].shape == (12,) + + def test_accepts_when_input_swapped_dims(self): + input_properties = { + 'input1': { + 'dims': ['dim1', 'dim2'], + 'units': 'm', + } + } + component = self.get_component(input_properties=input_properties) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.zeros([3, 4]), + dims=['dim2', 'dim1'], + attrs={'units': 'm'}, + ), + } + self.call_component(component, state) + assert component.state_given['input1'].shape == (4, 3) + + def test_input_requires_dims(self): + input_properties = {'input1': {'units': 'm'}} + with self.assertRaises(InvalidPropertyDictError): + self.get_component(input_properties=input_properties) + + def test_input_requires_units(self): + input_properties = {'input1': {'dims': ['dim1']}} + with self.assertRaises(InvalidPropertyDictError): + self.get_component(input_properties=input_properties) + + def test_input_no_transformations(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + component = self.get_component(input_properties=input_properties) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + self.call_component(component, state) + assert len(component.state_given) == 2 + assert 'time' in component.state_given.keys() + assert 'input1' in component.state_given.keys() + assert isinstance(component.state_given['input1'], np.ndarray) + assert np.all(component.state_given['input1'] == np.ones([10])) + + def test_input_converts_units(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + component = self.get_component(input_properties=input_properties) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'km'} + ) + } + self.call_component(component, state) + assert len(component.state_given) == 2 + assert 'time' in component.state_given.keys() + assert 'input1' in component.state_given.keys() + assert isinstance(component.state_given['input1'], np.ndarray) + assert np.all(component.state_given['input1'] == np.ones([10])*1000.) + + def test_input_collects_one_dimension(self): + input_properties = { + 'input1': { + 'dims': ['*'], + 'units': 'm' + } + } + component = self.get_component(input_properties=input_properties) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + self.call_component(component, state) + assert len(component.state_given) == 2 + assert 'time' in component.state_given.keys() + assert 'input1' in component.state_given.keys() + assert isinstance(component.state_given['input1'], np.ndarray) + assert np.all(component.state_given['input1'] == np.ones([10])) + + def test_input_is_aliased(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'in1', + } + } + component = self.get_component(input_properties=input_properties) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + self.call_component(component, state) + assert len(component.state_given) == 2 + assert 'time' in component.state_given.keys() + assert 'in1' in component.state_given.keys() + assert isinstance(component.state_given['in1'], np.ndarray) + assert np.all(component.state_given['in1'] == np.ones([10])) + + +class DiagnosticTestBase(): + + def test_diagnostic_requires_dims(self): + diagnostic_properties = {'diag1': {'units': 'm'}} + with self.assertRaises(InvalidPropertyDictError): + self.get_component(diagnostic_properties=diagnostic_properties) + + def test_diagnostic_requires_units(self): + diagnostic_properties = {'diag1': {'dims': ['dim1']}} + with self.assertRaises(InvalidPropertyDictError): + self.get_component(diagnostic_properties=diagnostic_properties) + + def test_diagnostic_requires_correct_number_of_dims(self): + input_properties = { + 'input1': {'units': 'm', 'dims': ['dim1', 'dim2']} + } + diagnostic_properties = { + 'diag1': {'units': 'm', 'dims': ['dim1', 'dim2']} + } + diagnostic_output = {'diag1': np.zeros([10]),} + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10, 2]), + dims=['dim1', 'dim2'], + attrs={'units': 'm'} + ) + } + component = self.get_component( + input_properties = input_properties, + diagnostic_properties=diagnostic_properties, + diagnostic_output=diagnostic_output, + ) + with self.assertRaises(InvalidPropertyDictError): + _, _ = self.call_component(component, state) + + def test_diagnostic_requires_correct_dim_length(self): + input_properties = { + 'input1': {'units': 'm', 'dims': ['dim1', 'dim2']} + } + diagnostic_properties = { + 'diag1': {'units': 'm', 'dims': ['dim1', 'dim2']} + } + diagnostic_output = {'diag1': np.zeros([5, 2]),} + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10, 2]), + dims=['dim1', 'dim2'], + attrs={'units': 'm'} + ) + } + component = self.get_component( + input_properties=input_properties, + diagnostic_properties=diagnostic_properties, + diagnostic_output=diagnostic_output + ) + with self.assertRaises(InvalidPropertyDictError): + _, _ = self.call_component(component, state) + + def test_diagnostic_uses_input_dims(self): + input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} + diagnostic_properties = {'diag1': {'units': 'm'}} + self.get_component( + input_properties=input_properties, + diagnostic_properties=diagnostic_properties + ) + + def test_diagnostic_doesnt_use_input_units(self): + input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} + diagnostic_properties = {'diag1': {'dims': ['dim1']}} + with self.assertRaises(InvalidPropertyDictError): + self.get_component( + input_properties=input_properties, + diagnostic_properties=diagnostic_properties + ) + + def test_diagnostics_no_transformations(self): + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm' + } + } + diagnostic_output = { + 'output1': np.ones([10]), + } + component = self.get_component( + diagnostic_properties=diagnostic_properties, + diagnostic_output=diagnostic_output, + ) + state = {'time': timedelta(0)} + diagnostics = self.get_diagnostics(self.call_component(component, state)) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_restoring_dims(self): + input_properties = { + 'input1': { + 'dims': ['*', 'dim1'], + 'units': 'm', + } + } + diagnostic_properties = { + 'output1': { + 'dims': ['*', 'dim1'], + 'units': 'm' + } + } + diagnostic_output = { + 'output1': np.ones([1, 10]), + } + component = self.get_component( + input_properties=input_properties, + diagnostic_properties=diagnostic_properties, + diagnostic_output=diagnostic_output, + ) + state = { + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'}), + 'time': timedelta(0)} + diagnostics = self.get_diagnostics(self.call_component(component, state)) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_with_alias(self): + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'out1', + } + } + diagnostic_output = { + 'out1': np.ones([10]), + } + component = self.get_component( + diagnostic_properties=diagnostic_properties, + diagnostic_output=diagnostic_output, + ) + state = {'time': timedelta(0)} + diagnostics = self.get_diagnostics(self.call_component(component, state)) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_with_alias_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + 'alias': 'out1', + } + } + diagnostic_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + diagnostic_output = { + 'out1': np.ones([10]), + } + component = self.get_component( + input_properties=input_properties, + diagnostic_properties=diagnostic_properties, + diagnostic_output=diagnostic_output, + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + diagnostics = self.get_diagnostics(self.call_component(component, state)) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_diagnostics_with_dims_from_input(self): + input_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + diagnostic_properties = { + 'output1': { + 'units': 'm', + } + } + diagnostic_output = { + 'output1': np.ones([10]), + } + component = self.get_component( + input_properties=input_properties, + diagnostic_properties=diagnostic_properties, + diagnostic_output=diagnostic_output, + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'} + ) + } + diagnostics = self.get_diagnostics(self.call_component(component, state)) + assert len(diagnostics) == 1 + assert 'output1' in diagnostics.keys() + assert isinstance(diagnostics['output1'], DataArray) + assert len(diagnostics['output1'].dims) == 1 + assert 'dim1' in diagnostics['output1'].dims + assert 'units' in diagnostics['output1'].attrs + assert diagnostics['output1'].attrs['units'] == 'm' + assert np.all(diagnostics['output1'].values == np.ones([10])) + + def test_raises_when_diagnostic_not_given(self): + diagnostic_properties = { + 'diag1': { + 'dims': ['dims1'], + 'units': 'm', + } + } + diagnostic_output = {} + diagnostic = self.get_component( + diagnostic_properties=diagnostic_properties, + diagnostic_output=diagnostic_output + ) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentMissingOutputError): + self.call_component(diagnostic, state) + + def test_raises_when_extraneous_diagnostic_given(self): + diagnostic_properties = {} + diagnostic_output = { + 'diag1': np.zeros([10]) + } + diagnostic = self.get_component( + diagnostic_properties=diagnostic_properties, + diagnostic_output=diagnostic_output + ) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentExtraOutputError): + self.call_component(diagnostic, state) -class PrognosticTests(unittest.TestCase): + +class PrognosticTests(unittest.TestCase, InputTestBase): component_class = MockPrognostic def call_component(self, component, state): return component(state) + def get_component( + self, input_properties=None, tendency_properties=None, + diagnostic_properties=None, tendency_output=None, + diagnostic_output=None): + return MockPrognostic( + input_properties=input_properties or {}, + tendency_properties=tendency_properties or {}, + diagnostic_properties=diagnostic_properties or {}, + tendency_output=tendency_output or {}, + diagnostic_output=diagnostic_output or {}, + ) + + def get_diagnostics(self, result): + return result[1] + def test_cannot_use_bad_component(self): component = BadMockPrognostic() with self.assertRaises(RuntimeError): @@ -254,134 +787,6 @@ def test_empty_prognostic(self): assert prognostic.state_given['time'] == timedelta(seconds=0) assert prognostic.times_called == 1 - def test_input_requires_dims(self): - input_properties = {'input1': {'units': 'm'}} - diagnostic_properties = {} - tendency_properties = {} - diagnostic_output = {} - tendency_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - - def test_input_requires_units(self): - input_properties = {'input1': {'dims': ['dim1']}} - diagnostic_properties = {} - tendency_properties = {} - diagnostic_output = {} - tendency_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - tendency_properties, - diagnostic_output, tendency_output - ) - - def test_diagnostic_requires_dims(self): - input_properties = {} - diagnostic_properties = {'diag1': {'units': 'm'}} - tendency_properties = {} - diagnostic_output = {} - tendency_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - tendency_properties, - diagnostic_output, tendency_output - ) - - def test_diagnostic_requires_correct_number_of_dims(self): - input_properties = { - 'input1': {'units': 'm', 'dims': ['dim1', 'dim2']} - } - diagnostic_properties = { - 'diag1': {'units': 'm', 'dims': ['dim1', 'dim2']} - } - tendency_properties = {} - diagnostic_output = {'diag1': np.zeros([10]),} - tendency_output = {} - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10, 2]), - dims=['dim1', 'dim2'], - attrs={'units': 'm'} - ) - } - component = self.component_class( - input_properties, diagnostic_properties, - tendency_properties, - diagnostic_output, tendency_output - ) - with self.assertRaises(InvalidPropertyDictError): - _, _ = self.call_component(component, state) - - def test_diagnostic_requires_correct_dim_length(self): - input_properties = { - 'input1': {'units': 'm', 'dims': ['dim1', 'dim2']} - } - diagnostic_properties = { - 'diag1': {'units': 'm', 'dims': ['dim1', 'dim2']} - } - tendency_properties = {} - diagnostic_output = {'diag1': np.zeros([5, 2]),} - tendency_output = {} - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10, 2]), - dims=['dim1', 'dim2'], - attrs={'units': 'm'} - ) - } - component = self.component_class( - input_properties, diagnostic_properties, - tendency_properties, - diagnostic_output, tendency_output - ) - with self.assertRaises(InvalidPropertyDictError): - _, _ = self.call_component(component, state) - - def test_diagnostic_uses_base_dims(self): - input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} - diagnostic_properties = {'diag1': {'units': 'm'}} - tendency_properties = {} - diagnostic_output = {} - tendency_output = {} - self.component_class( - input_properties, diagnostic_properties, - tendency_properties, - diagnostic_output, tendency_output - ) - - def test_diagnostic_doesnt_use_base_units(self): - input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} - diagnostic_properties = {'diag1': {'dims': ['dim1']}} - tendency_properties = {} - diagnostic_output = {} - tendency_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - tendency_properties, - diagnostic_output, tendency_output - ) - - def test_diagnostic_requires_units(self): - input_properties = {} - diagnostic_properties = {'diag1': {'dims': ['dim1']}} - tendency_properties = {} - diagnostic_output = {} - tendency_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - tendency_properties, - diagnostic_output, tendency_output - ) - def test_tendency_requires_dims(self): input_properties = {} diagnostic_properties = {} @@ -498,130 +903,35 @@ def test_cannot_overlap_tendency_aliases(self): tendency_output = {} with self.assertRaises(InvalidPropertyDictError): self.component_class( - input_properties, diagnostic_properties, - tendency_properties, - diagnostic_output, tendency_output - ) - - def test_raises_when_extraneous_tendency_given(self): - input_properties = {} - diagnostic_properties = {} - tendency_properties = {} - diagnostic_output = {} - tendency_output = { - 'tend1': np.zeros([10]), - } - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - state = {'time': timedelta(0)} - with self.assertRaises(ComponentExtraOutputError): - _, _ = self.call_component(prognostic, state) - - def test_raises_when_diagnostic_not_given(self): - input_properties = {} - diagnostic_properties = { - 'diag1': { - 'dims': ['dims1'], - 'units': 'm', - } - } - tendency_properties = {} - diagnostic_output = {} - tendency_output = {} - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - state = {'time': timedelta(0)} - with self.assertRaises(ComponentMissingOutputError): - _, _ = self.call_component(prognostic, state) - - def test_raises_when_extraneous_diagnostic_given(self): - input_properties = {} - diagnostic_properties = {} - tendency_properties = {} - diagnostic_output = { - 'diag1': np.zeros([10]) - } - tendency_output = {} - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - state = {'time': timedelta(0)} - with self.assertRaises(ComponentExtraOutputError): - _, _ = self.call_component(prognostic, state) - - def test_input_no_transformations(self): - input_properties = { - 'input1': { - 'dims': ['dim1'], - 'units': 'm' - } - } - diagnostic_properties = {} - tendency_properties = {} - diagnostic_output = {} - tendency_output = {} - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} + input_properties, diagnostic_properties, + tendency_properties, + diagnostic_output, tendency_output ) - } - _, _ = self.call_component(prognostic, state) - assert len(prognostic.state_given) == 2 - assert 'time' in prognostic.state_given.keys() - assert 'input1' in prognostic.state_given.keys() - assert isinstance(prognostic.state_given['input1'], np.ndarray) - assert np.all(prognostic.state_given['input1'] == np.ones([10])) - def test_input_converts_units(self): - input_properties = { - 'input1': { - 'dims': ['dim1'], - 'units': 'm' - } - } + def test_raises_when_extraneous_tendency_given(self): + input_properties = {} diagnostic_properties = {} tendency_properties = {} diagnostic_output = {} - tendency_output = {} + tendency_output = { + 'tend1': np.zeros([10]), + } prognostic = self.component_class( input_properties, diagnostic_properties, tendency_properties, diagnostic_output, tendency_output ) - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'km'} - ) - } - _, _ = self.call_component(prognostic, state) - assert len(prognostic.state_given) == 2 - assert 'time' in prognostic.state_given.keys() - assert 'input1' in prognostic.state_given.keys() - assert isinstance(prognostic.state_given['input1'], np.ndarray) - assert np.all(prognostic.state_given['input1'] == np.ones([10])*1000.) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentExtraOutputError): + _, _ = self.call_component(prognostic, state) - def test_input_collects_one_dimension(self): - input_properties = { - 'input1': { - 'dims': ['*'], - 'units': 'm' + def test_raises_when_diagnostic_not_given(self): + input_properties = {} + diagnostic_properties = { + 'diag1': { + 'dims': ['dims1'], + 'units': 'm', } } - diagnostic_properties = {} tendency_properties = {} diagnostic_output = {} tendency_output = {} @@ -629,51 +939,25 @@ def test_input_collects_one_dimension(self): input_properties, diagnostic_properties, tendency_properties, diagnostic_output, tendency_output ) - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - _, _ = self.call_component(prognostic, state) - assert len(prognostic.state_given) == 2 - assert 'time' in prognostic.state_given.keys() - assert 'input1' in prognostic.state_given.keys() - assert isinstance(prognostic.state_given['input1'], np.ndarray) - assert np.all(prognostic.state_given['input1'] == np.ones([10])) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentMissingOutputError): + _, _ = self.call_component(prognostic, state) - def test_input_is_aliased(self): - input_properties = { - 'input1': { - 'dims': ['dim1'], - 'units': 'm', - 'alias': 'in1', - } - } + def test_raises_when_extraneous_diagnostic_given(self): + input_properties = {} diagnostic_properties = {} tendency_properties = {} - diagnostic_output = {} + diagnostic_output = { + 'diag1': np.zeros([10]) + } tendency_output = {} prognostic = self.component_class( input_properties, diagnostic_properties, tendency_properties, diagnostic_output, tendency_output ) - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - _, _ = self.call_component(prognostic, state) - assert len(prognostic.state_given) == 2 - assert 'time' in prognostic.state_given.keys() - assert 'in1' in prognostic.state_given.keys() - assert isinstance(prognostic.state_given['in1'], np.ndarray) - assert np.all(prognostic.state_given['in1'] == np.ones([10])) + state = {'time': timedelta(0)} + with self.assertRaises(ComponentExtraOutputError): + _, _ = self.call_component(prognostic, state) def test_tendencies_no_transformations(self): input_properties = {} @@ -778,191 +1062,16 @@ def test_tendencies_with_dims_from_input(self): 'units': 'm', } } - diagnostic_properties = {} - tendency_properties = { - 'output1': { - 'units': 'm/s', - } - } - diagnostic_output = {} - tendency_output = { - 'output1': np.ones([10]), - } - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - state = { - 'time': timedelta(0), - 'output1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - tendencies, _ = self.call_component(prognostic, state) - assert len(tendencies) == 1 - assert 'output1' in tendencies.keys() - assert isinstance(tendencies['output1'], DataArray) - assert len(tendencies['output1'].dims) == 1 - assert 'dim1' in tendencies['output1'].dims - assert 'units' in tendencies['output1'].attrs - assert tendencies['output1'].attrs['units'] == 'm/s' - assert np.all(tendencies['output1'].values == np.ones([10])) - - def test_diagnostics_no_transformations(self): - input_properties = {} - diagnostic_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm' - } - } - tendency_properties = {} - diagnostic_output = { - 'output1': np.ones([10]), - } - tendency_output = {} - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - state = {'time': timedelta(0)} - _, diagnostics = self.call_component(prognostic, state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) - - def test_diagnostics_restoring_dims(self): - input_properties = { - 'input1': { - 'dims': ['*', 'dim1'], - 'units': 'm', - } - } - diagnostic_properties = { - 'output1': { - 'dims': ['*', 'dim1'], - 'units': 'm' - } - } - tendency_properties = {} - diagnostic_output = { - 'output1': np.ones([1, 10]), - } - tendency_output = {} - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - state = { - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'}), - 'time': timedelta(0)} - _, diagnostics = self.call_component(prognostic, state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) - - def test_diagnostics_with_alias(self): - input_properties = {} - diagnostic_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - 'alias': 'out1', - } - } - tendency_properties = {} - diagnostic_output = { - 'out1': np.ones([10]), - } - tendency_output = {} - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - state = {'time': timedelta(0)} - _, diagnostics = self.call_component(prognostic, state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) - - def test_diagnostics_with_alias_from_input(self): - input_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - 'alias': 'out1', - } - } - diagnostic_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - } - } - tendency_properties = {} - diagnostic_output = { - 'out1': np.ones([10]), - } - tendency_output = {} - prognostic = self.component_class( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - state = { - 'time': timedelta(0), - 'output1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - _, diagnostics = self.call_component(prognostic, state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) - - def test_diagnostics_with_dims_from_input(self): - input_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - } - } - diagnostic_properties = { + diagnostic_properties = {} + tendency_properties = { 'output1': { - 'units': 'm', + 'units': 'm/s', } } - tendency_properties = {} - diagnostic_output = { + diagnostic_output = {} + tendency_output = { 'output1': np.ones([10]), } - tendency_output = {} prognostic = self.component_class( input_properties, diagnostic_properties, tendency_properties, diagnostic_output, tendency_output @@ -975,15 +1084,15 @@ def test_diagnostics_with_dims_from_input(self): attrs={'units': 'm'} ) } - _, diagnostics = self.call_component(prognostic, state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) + tendencies, _ = self.call_component(prognostic, state) + assert len(tendencies) == 1 + assert 'output1' in tendencies.keys() + assert isinstance(tendencies['output1'], DataArray) + assert len(tendencies['output1'].dims) == 1 + assert 'dim1' in tendencies['output1'].dims + assert 'units' in tendencies['output1'].attrs + assert tendencies['output1'].attrs['units'] == 'm/s' + assert np.all(tendencies['output1'].values == np.ones([10])) def test_tendencies_in_diagnostics_no_tendency(self): input_properties = {} @@ -1122,6 +1231,18 @@ class ImplicitPrognosticTests(PrognosticTests): def call_component(self, component, state): return component(state, timedelta(seconds=1)) + def get_component( + self, input_properties=None, tendency_properties=None, + diagnostic_properties=None, tendency_output=None, + diagnostic_output=None): + return MockImplicitPrognostic( + input_properties=input_properties or {}, + tendency_properties=tendency_properties or {}, + diagnostic_properties=diagnostic_properties or {}, + tendency_output=tendency_output or {}, + diagnostic_output=diagnostic_output or {}, + ) + def test_cannot_use_bad_component(self): component = BadMockImplicitPrognostic() with self.assertRaises(RuntimeError): @@ -1231,13 +1352,26 @@ def test_timedelta_is_passed(self): assert prognostic.times_called == 1 -class DiagnosticTests(unittest.TestCase): +class DiagnosticTests(unittest.TestCase, InputTestBase, DiagnosticTestBase): component_class = MockDiagnostic def call_component(self, component, state): return component(state) + def get_component( + self, input_properties=None, + diagnostic_properties=None, + diagnostic_output=None): + return MockDiagnostic( + input_properties=input_properties or {}, + diagnostic_properties=diagnostic_properties or {}, + diagnostic_output=diagnostic_output or {}, + ) + + def get_diagnostics(self, result): + return result + def test_cannot_use_bad_component(self): component = BadMockDiagnostic() with self.assertRaises(RuntimeError): @@ -1286,389 +1420,46 @@ def array_call(self, state): class MyDiagnostic2(Diagnostic): input_properties = {} - diagnostic_properties = {} - - def array_call(self, state): - pass - - diag1 = MyDiagnostic1() - assert not isinstance(diag1, MyDiagnostic2) - - def test_empty_diagnostic(self): - diagnostic = self.component_class({}, {}, {}) - diagnostics = diagnostic({'time': timedelta(seconds=0)}) - assert diagnostics == {} - assert len(diagnostic.state_given) == 1 - assert 'time' in diagnostic.state_given.keys() - assert diagnostic.state_given['time'] == timedelta(seconds=0) - assert diagnostic.times_called == 1 - - def test_input_requires_dims(self): - input_properties = {'input1': {'units': 'm'}} - diagnostic_properties = {} - diagnostic_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - diagnostic_output - ) - - def test_input_requires_units(self): - input_properties = {'input1': {'dims': ['dim1']}} - diagnostic_properties = {} - diagnostic_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - diagnostic_output - ) - - def test_diagnostic_requires_dims(self): - input_properties = {} - diagnostic_properties = {'diag1': {'units': 'm'}} - diagnostic_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - diagnostic_output, - ) - - def test_diagnostic_uses_base_dims(self): - input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} - diagnostic_properties = {'diag1': {'units': 'm'}} - diagnostic_output = {} - self.component_class( - input_properties, diagnostic_properties, - diagnostic_output, - ) - - def test_diagnostic_doesnt_use_base_units(self): - input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} - diagnostic_properties = {'diag1': {'dims': ['dim1']}} - diagnostic_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - diagnostic_output, - ) - - def test_diagnostic_requires_units(self): - input_properties = {} - diagnostic_properties = {'diag1': {'dims': ['dim1']}} - diagnostic_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - diagnostic_output, - ) - - def test_cannot_overlap_input_aliases(self): - input_properties = { - 'input1': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'}, - 'input2': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'} - } - diagnostic_properties = {} - diagnostic_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - diagnostic_output - ) - - def test_cannot_overlap_diagnostic_aliases(self): - input_properties = { - } - diagnostic_properties = { - 'diag1': {'dims': ['dim1'], 'units': 'm', 'alias': 'diag'}, - 'diag2': {'dims': ['dim1'], 'units': 'm', 'alias': 'diag'} - } - diagnostic_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - diagnostic_output - ) - - def test_raises_when_diagnostic_not_given(self): - input_properties = {} - diagnostic_properties = { - 'diag1': { - 'dims': ['dims1'], - 'units': 'm', - } - } - diagnostic_output = {} - diagnostic = self.component_class( - input_properties, diagnostic_properties, - diagnostic_output, - ) - state = {'time': timedelta(0)} - with self.assertRaises(ComponentMissingOutputError): - _, _ = self.call_component(diagnostic, state) - - def test_raises_when_extraneous_diagnostic_given(self): - input_properties = {} - diagnostic_properties = {} - diagnostic_output = { - 'diag1': np.zeros([10]) - } - diagnostic = self.component_class( - input_properties, diagnostic_properties, - diagnostic_output, - ) - state = {'time': timedelta(0)} - with self.assertRaises(ComponentExtraOutputError): - _, _ = self.call_component(diagnostic, state) - - def test_input_no_transformations(self): - input_properties = { - 'input1': { - 'dims': ['dim1'], - 'units': 'm' - } - } - diagnostic_properties = {} - diagnostic_output = {} - diagnostic = MockDiagnostic( - input_properties, diagnostic_properties, diagnostic_output - ) - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - _ = diagnostic(state) - assert len(diagnostic.state_given) == 2 - assert 'time' in diagnostic.state_given.keys() - assert 'input1' in diagnostic.state_given.keys() - assert isinstance(diagnostic.state_given['input1'], np.ndarray) - assert np.all(diagnostic.state_given['input1'] == np.ones([10])) - - def test_input_converts_units(self): - input_properties = { - 'input1': { - 'dims': ['dim1'], - 'units': 'm' - } - } - diagnostic_properties = {} - diagnostic_output = {} - diagnostic = MockDiagnostic( - input_properties, diagnostic_properties, - diagnostic_output - ) - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'km'} - ) - } - _ = diagnostic(state) - assert len(diagnostic.state_given) == 2 - assert 'time' in diagnostic.state_given.keys() - assert 'input1' in diagnostic.state_given.keys() - assert isinstance(diagnostic.state_given['input1'], np.ndarray) - assert np.all(diagnostic.state_given['input1'] == np.ones([10])*1000.) - - def test_input_collects_one_dimension(self): - input_properties = { - 'input1': { - 'dims': ['*'], - 'units': 'm' - } - } - diagnostic_properties = {} - diagnostic_output = {} - diagnostic = MockDiagnostic( - input_properties, diagnostic_properties, - diagnostic_output - ) - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - _ = diagnostic(state) - assert len(diagnostic.state_given) == 2 - assert 'time' in diagnostic.state_given.keys() - assert 'input1' in diagnostic.state_given.keys() - assert isinstance(diagnostic.state_given['input1'], np.ndarray) - assert np.all(diagnostic.state_given['input1'] == np.ones([10])) - - def test_input_is_aliased(self): - input_properties = { - 'input1': { - 'dims': ['dim1'], - 'units': 'm', - 'alias': 'in1', - } - } - diagnostic_properties = {} - diagnostic_output = {} - diagnostic = MockDiagnostic( - input_properties, diagnostic_properties, - diagnostic_output - ) - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - _ = diagnostic(state) - assert len(diagnostic.state_given) == 2 - assert 'time' in diagnostic.state_given.keys() - assert 'in1' in diagnostic.state_given.keys() - assert isinstance(diagnostic.state_given['in1'], np.ndarray) - assert np.all(diagnostic.state_given['in1'] == np.ones([10])) - - def test_diagnostics_no_transformations(self): - input_properties = {} - diagnostic_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm' - } - } - diagnostic_output = { - 'output1': np.ones([10]), - } - diagnostic = MockDiagnostic( - input_properties, diagnostic_properties, - diagnostic_output - ) - state = {'time': timedelta(0)} - diagnostics = diagnostic(state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) + diagnostic_properties = {} - def test_diagnostics_with_alias(self): - input_properties = {} - diagnostic_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - 'alias': 'out1', - } - } - diagnostic_output = { - 'out1': np.ones([10]), - } - diagnostic = MockDiagnostic( - input_properties, diagnostic_properties, - diagnostic_output - ) - state = {'time': timedelta(0)} - diagnostics = diagnostic(state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) + def array_call(self, state): + pass - def test_diagnostics_with_alias_from_input(self): - input_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - 'alias': 'out1', - } - } - diagnostic_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - } - } - diagnostic_output = { - 'out1': np.ones([10]), - } - diagnostic = MockDiagnostic( - input_properties, diagnostic_properties, - diagnostic_output - ) - state = { - 'time': timedelta(0), - 'output1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - diagnostics = diagnostic(state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) + diag1 = MyDiagnostic1() + assert not isinstance(diag1, MyDiagnostic2) - def test_diagnostics_with_dims_from_input(self): - input_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - } - } - diagnostic_properties = { - 'output1': { - 'units': 'm', - } - } - diagnostic_output = { - 'output1': np.ones([10]), - } - diagnostic = MockDiagnostic( - input_properties, diagnostic_properties, - diagnostic_output - ) - state = { - 'time': timedelta(0), - 'output1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - diagnostics = diagnostic(state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) + def test_empty_diagnostic(self): + diagnostic = self.component_class({}, {}, {}) + diagnostics = diagnostic({'time': timedelta(seconds=0)}) + assert diagnostics == {} + assert len(diagnostic.state_given) == 1 + assert 'time' in diagnostic.state_given.keys() + assert diagnostic.state_given['time'] == timedelta(seconds=0) + assert diagnostic.times_called == 1 -class ImplicitTests(unittest.TestCase): +class ImplicitTests(unittest.TestCase, InputTestBase, DiagnosticTestBase): component_class = MockImplicit def call_component(self, component, state): return component(state, timedelta(seconds=1)) + def get_component( + self, input_properties=None, output_properties=None, + diagnostic_properties=None, state_output=None, + diagnostic_output=None): + return MockImplicit( + input_properties=input_properties or {}, + output_properties=output_properties or {}, + diagnostic_properties=diagnostic_properties or {}, + state_output=state_output or {}, + diagnostic_output=diagnostic_output or {}, + ) + + def get_diagnostics(self, result): + return result[0] + def test_cannot_use_bad_component(self): component = BadMockImplicit() with self.assertRaises(RuntimeError): @@ -1752,83 +1543,6 @@ def test_empty_implicit(self): assert implicit.state_given['time'] == timedelta(seconds=0) assert implicit.times_called == 1 - def test_input_requires_dims(self): - input_properties = {'input1': {'units': 'm'}} - diagnostic_properties = {} - output_properties = {} - diagnostic_output = {} - state_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - output_properties, - diagnostic_output, state_output - ) - - def test_input_requires_units(self): - input_properties = {'input1': {'dims': ['dim1']}} - diagnostic_properties = {} - output_properties = {} - diagnostic_output = {} - state_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - output_properties, - diagnostic_output, state_output - ) - - def test_diagnostic_requires_dims(self): - input_properties = {} - diagnostic_properties = {'diag1': {'units': 'm'}} - output_properties = {} - diagnostic_output = {} - state_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - output_properties, - diagnostic_output, state_output - ) - - def test_diagnostic_uses_base_dims(self): - input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} - diagnostic_properties = {'diag1': {'units': 'm'}} - output_properties = {} - diagnostic_output = {} - state_output = {} - self.component_class( - input_properties, diagnostic_properties, - output_properties, - diagnostic_output, state_output - ) - - def test_diagnostic_doesnt_use_base_units(self): - input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} - diagnostic_properties = {'diag1': {'dims': ['dim1']}} - output_properties = {} - diagnostic_output = {} - state_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - output_properties, - diagnostic_output, state_output - ) - - def test_diagnostic_requires_units(self): - input_properties = {} - diagnostic_properties = {'diag1': {'dims': ['dim1']}} - output_properties = {} - diagnostic_output = {} - state_output = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - output_properties, - diagnostic_output, state_output - ) - def test_output_requires_dims(self): input_properties = {} diagnostic_properties = {} @@ -1880,39 +1594,6 @@ def test_output_requires_units(self): diagnostic_output, state_output ) - def test_cannot_overlap_input_aliases(self): - input_properties = { - 'input1': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'}, - 'input2': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'} - } - diagnostic_properties = {} - output_properties = {} - diagnostic_output = {} - output_state = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - output_properties, - diagnostic_output, output_state - ) - - def test_cannot_overlap_diagnostic_aliases(self): - input_properties = { - } - diagnostic_properties = { - 'diag1': {'dims': ['dim1'], 'units': 'm', 'alias': 'diag'}, - 'diag2': {'dims': ['dim1'], 'units': 'm', 'alias': 'diag'} - } - output_properties = {} - diagnostic_output = {} - output_state = {} - with self.assertRaises(InvalidPropertyDictError): - self.component_class( - input_properties, diagnostic_properties, - output_properties, - diagnostic_output, output_state - ) - def test_cannot_overlap_output_aliases(self): input_properties = { } @@ -1933,57 +1614,22 @@ def test_cannot_overlap_output_aliases(self): def test_timedelta_is_passed(self): implicit = MockImplicit({}, {}, {}, {}, {}) - tendencies, diagnostics = implicit( - {'time': timedelta(seconds=0)}, timedelta(seconds=5)) - assert tendencies == {} - assert diagnostics == {} - assert implicit.timestep_given == timedelta(seconds=5) - assert implicit.times_called == 1 - - def test_raises_when_output_not_given(self): - input_properties = {} - diagnostic_properties = {} - output_properties = { - 'output1': { - 'dims': ['dims1'], - 'units': 'm', - } - } - diagnostic_output = {} - state_output = {} - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, state_output - ) - state = {'time': timedelta(0)} - with self.assertRaises(ComponentMissingOutputError): - _, _ = self.call_component(implicit, state) - - def test_raises_when_extraneous_output_given(self): - input_properties = {} - diagnostic_properties = {} - output_properties = {} - diagnostic_output = {} - state_output = { - 'tend1': np.zeros([10]), - } - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, state_output - ) - state = {'time': timedelta(0)} - with self.assertRaises(ComponentExtraOutputError): - _, _ = self.call_component(implicit, state) + tendencies, diagnostics = implicit( + {'time': timedelta(seconds=0)}, timedelta(seconds=5)) + assert tendencies == {} + assert diagnostics == {} + assert implicit.timestep_given == timedelta(seconds=5) + assert implicit.times_called == 1 - def test_raises_when_diagnostic_not_given(self): + def test_raises_when_output_not_given(self): input_properties = {} - diagnostic_properties = { - 'diag1': { + diagnostic_properties = {} + output_properties = { + 'output1': { 'dims': ['dims1'], 'units': 'm', } } - output_properties = {} diagnostic_output = {} state_output = {} implicit = self.component_class( @@ -1994,14 +1640,14 @@ def test_raises_when_diagnostic_not_given(self): with self.assertRaises(ComponentMissingOutputError): _, _ = self.call_component(implicit, state) - def test_raises_when_extraneous_diagnostic_given(self): + def test_raises_when_extraneous_output_given(self): input_properties = {} diagnostic_properties = {} output_properties = {} - diagnostic_output = { - 'diag1': np.zeros([10]) + diagnostic_output = {} + state_output = { + 'tend1': np.zeros([10]), } - state_output = {} implicit = self.component_class( input_properties, diagnostic_properties, output_properties, diagnostic_output, state_output @@ -2010,127 +1656,6 @@ def test_raises_when_extraneous_diagnostic_given(self): with self.assertRaises(ComponentExtraOutputError): _, _ = self.call_component(implicit, state) - def test_input_no_transformations(self): - input_properties = { - 'input1': { - 'dims': ['dim1'], - 'units': 'm' - } - } - diagnostic_properties = {} - output_properties = {} - diagnostic_output = {} - output_state = {} - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, output_state - ) - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - _, _ = self.call_component(implicit, state) - assert len(implicit.state_given) == 2 - assert 'time' in implicit.state_given.keys() - assert 'input1' in implicit.state_given.keys() - assert isinstance(implicit.state_given['input1'], np.ndarray) - assert np.all(implicit.state_given['input1'] == np.ones([10])) - - def test_input_converts_units(self): - input_properties = { - 'input1': { - 'dims': ['dim1'], - 'units': 'm' - } - } - diagnostic_properties = {} - output_properties = {} - diagnostic_output = {} - output_state = {} - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, output_state - ) - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'km'} - ) - } - _, _ = self.call_component(implicit, state) - assert len(implicit.state_given) == 2 - assert 'time' in implicit.state_given.keys() - assert 'input1' in implicit.state_given.keys() - assert isinstance(implicit.state_given['input1'], np.ndarray) - assert np.all(implicit.state_given['input1'] == np.ones([10])*1000.) - - def test_input_collects_one_dimension(self): - input_properties = { - 'input1': { - 'dims': ['*'], - 'units': 'm' - } - } - diagnostic_properties = {} - output_properties = {} - diagnostic_output = {} - output_state = {} - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, output_state - ) - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - _, _ = self.call_component(implicit, state) - assert len(implicit.state_given) == 2 - assert 'time' in implicit.state_given.keys() - assert 'input1' in implicit.state_given.keys() - assert isinstance(implicit.state_given['input1'], np.ndarray) - assert np.all(implicit.state_given['input1'] == np.ones([10])) - - def test_input_is_aliased(self): - input_properties = { - 'input1': { - 'dims': ['dim1'], - 'units': 'm', - 'alias': 'in1', - } - } - diagnostic_properties = {} - output_properties = {} - diagnostic_output = {} - output_state = {} - prognostic = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, output_state - ) - state = { - 'time': timedelta(0), - 'input1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - _, _ = self.call_component(prognostic, state) - assert len(prognostic.state_given) == 2 - assert 'time' in prognostic.state_given.keys() - assert 'in1' in prognostic.state_given.keys() - assert isinstance(prognostic.state_given['in1'], np.ndarray) - assert np.all(prognostic.state_given['in1'] == np.ones([10])) - def test_output_no_transformations(self): input_properties = {} diagnostic_properties = {} @@ -2266,143 +1791,6 @@ def test_output_with_dims_from_input(self): assert output['output1'].attrs['units'] == 'm/s' assert np.all(output['output1'].values == np.ones([10])) - def test_diagnostics_no_transformations(self): - input_properties = {} - diagnostic_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm' - } - } - output_properties = {} - diagnostic_output = { - 'output1': np.ones([10]), - } - output_state = {} - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, output_state - ) - state = {'time': timedelta(0)} - diagnostics, _ = self.call_component(implicit, state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) - - def test_diagnostics_with_alias(self): - input_properties = {} - diagnostic_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - 'alias': 'out1', - } - } - output_properties = {} - diagnostic_output = { - 'out1': np.ones([10]), - } - output_state = {} - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, output_state - ) - state = {'time': timedelta(0)} - diagnostics, _ = self.call_component(implicit, state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) - - def test_diagnostics_with_alias_from_input(self): - input_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - 'alias': 'out1', - } - } - diagnostic_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - } - } - output_properties = {} - diagnostic_output = { - 'out1': np.ones([10]), - } - output_state = {} - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, output_state - ) - state = { - 'time': timedelta(0), - 'output1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - diagnostics, _ = self.call_component(implicit, state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) - - def test_diagnostics_with_dims_from_input(self): - input_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm', - } - } - diagnostic_properties = { - 'output1': { - 'units': 'm', - } - } - output_properties = {} - diagnostic_output = { - 'output1': np.ones([10]), - } - output_state = {} - implicit = self.component_class( - input_properties, diagnostic_properties, output_properties, - diagnostic_output, output_state - ) - state = { - 'time': timedelta(0), - 'output1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'} - ) - } - diagnostics, _ = self.call_component(implicit, state) - assert len(diagnostics) == 1 - assert 'output1' in diagnostics.keys() - assert isinstance(diagnostics['output1'], DataArray) - assert len(diagnostics['output1'].dims) == 1 - assert 'dim1' in diagnostics['output1'].dims - assert 'units' in diagnostics['output1'].attrs - assert diagnostics['output1'].attrs['units'] == 'm' - assert np.all(diagnostics['output1'].values == np.ones([10])) - def test_tendencies_in_diagnostics_no_tendency(self): input_properties = {} diagnostic_properties = {} diff --git a/tests/test_state.py b/tests/test_state.py new file mode 100644 index 0000000..999afc5 --- /dev/null +++ b/tests/test_state.py @@ -0,0 +1,313 @@ +import unittest +from sympl import initialize_numpy_arrays_with_properties, DataArray +import numpy as np + + +class InitializeNumpyArraysWithPropertiesTests(unittest.TestCase): + + def test_empty(self): + output_properties = { + } + input_properties = { + } + input_state = { + } + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert result == {} + + def test_single_output_single_dim(self): + output_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 's^-1', + } + } + input_state = { + 'input1': DataArray( + np.zeros([10]), + dims=['dim1'], + attrs={'units': 's^-1'} + ) + } + + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert len(result.keys()) == 1 + assert 'output1' in result.keys() + assert result['output1'].shape == (10,) + assert np.all(result['output1'] == np.zeros([10])) + + def test_single_output_two_dims(self): + output_properties = { + 'output1': { + 'dims': ['dim1', 'dim2'], + 'units': 'm', + } + } + input_properties = { + 'input1': { + 'dims': ['dim1', 'dim2'], + 'units': 's^-1', + } + } + input_state = { + 'input1': DataArray( + np.zeros([3, 7]), + dims=['dim1', 'dim2'], + attrs={'units': 's^-1'} + ) + } + + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert len(result.keys()) == 1 + assert 'output1' in result.keys() + assert result['output1'].shape == (3, 7) + assert np.all(result['output1'] == np.zeros([3, 7])) + + def test_single_output_two_dims_opposite_order(self): + output_properties = { + 'output1': { + 'dims': ['dim2', 'dim1'], + 'units': 'm', + } + } + input_properties = { + 'input1': { + 'dims': ['dim1', 'dim2'], + 'units': 's^-1', + } + } + input_state = { + 'input1': DataArray( + np.zeros([3, 7]), + dims=['dim1', 'dim2'], + attrs={'units': 's^-1'} + ) + } + + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert len(result.keys()) == 1 + assert 'output1' in result.keys() + assert result['output1'].shape == (7, 3) + assert np.all(result['output1'] == np.zeros([7, 3])) + + def test_two_outputs(self): + output_properties = { + 'output1': { + 'dims': ['dim2', 'dim1'], + 'units': 'm', + }, + 'output2': { + 'dims': ['dim1', 'dim2'], + 'units': 'm', + } + } + input_properties = { + 'input1': { + 'dims': ['dim1', 'dim2'], + 'units': 's^-1', + } + } + input_state = { + 'input1': DataArray( + np.zeros([3, 7]), + dims=['dim1', 'dim2'], + attrs={'units': 's^-1'} + ) + } + + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert len(result.keys()) == 2 + assert 'output1' in result.keys() + assert result['output1'].shape == (7, 3) + assert np.all(result['output1'] == np.zeros([7, 3])) + assert 'output2' in result.keys() + assert result['output2'].shape == (3, 7) + assert np.all(result['output2'] == np.zeros([3, 7])) + + def test_two_inputs(self): + output_properties = { + 'output1': { + 'dims': ['dim2', 'dim1'], + 'units': 'm', + }, + } + input_properties = { + 'input1': { + 'dims': ['dim1', 'dim2'], + 'units': 's^-1', + }, + 'input2': { + 'dims': ['dim2', 'dim1'], + 'units': 's^-1', + } + } + input_state = { + 'input1': DataArray( + np.zeros([3, 7]), + dims=['dim1', 'dim2'], + attrs={'units': 's^-1'} + ), + 'input2': DataArray( + np.zeros([7, 3]), + dims=['dim2', 'dim1'], + attrs={'units': 's^-1'} + ) + } + + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert len(result.keys()) == 1 + assert 'output1' in result.keys() + assert result['output1'].shape == (7, 3) + assert np.all(result['output1'] == np.zeros([7, 3])) + + def test_single_dim_wildcard(self): + output_properties = { + 'output1': { + 'dims': ['*'], + 'units': 'm', + } + } + input_properties = { + 'input1': { + 'dims': ['*'], + 'units': 's^-1', + } + } + input_state = { + 'input1': DataArray( + np.zeros([10]), + dims=['dim1'], + attrs={'units': 's^-1'} + ) + } + + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert len(result.keys()) == 1 + assert 'output1' in result.keys() + assert result['output1'].shape == (10,) + assert np.all(result['output1'] == np.zeros([10])) + + def test_two_dims_in_wildcard(self): + output_properties = { + 'output1': { + 'dims': ['*'], + 'units': 'm', + } + } + input_properties = { + 'input1': { + 'dims': ['*'], + 'units': 's^-1', + } + } + input_state = { + 'input1': DataArray( + np.zeros([4, 3]), + dims=['dim1', 'dim2'], + attrs={'units': 's^-1'} + ) + } + + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert len(result.keys()) == 1 + assert 'output1' in result.keys() + assert result['output1'].shape == (12,) + assert np.all(result['output1'] == np.zeros([12])) + + def test_two_dims_in_wildcard_with_basic_dim(self): + output_properties = { + 'output1': { + 'dims': ['*', 'dim3'], + 'units': 'm', + } + } + input_properties = { + 'input1': { + 'dims': ['*', 'dim3'], + 'units': 's^-1', + } + } + input_state = { + 'input1': DataArray( + np.zeros([4, 3, 2]), + dims=['dim1', 'dim2', 'dim3'], + attrs={'units': 's^-1'} + ) + } + + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert len(result.keys()) == 1 + assert 'output1' in result.keys() + assert result['output1'].shape == (12, 2) + assert np.all(result['output1'] == np.zeros([12, 2])) + + def test_two_dims_in_wildcard_with_basic_dim_in_center(self): + output_properties = { + 'output1': { + 'dims': ['*', 'dim2'], + 'units': 'm', + } + } + input_properties = { + 'input1': { + 'dims': ['*', 'dim2'], + 'units': 's^-1', + } + } + input_state = { + 'input1': DataArray( + np.zeros([4, 3, 2]), + dims=['dim1', 'dim2', 'dim3'], + attrs={'units': 's^-1'} + ) + } + + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert len(result.keys()) == 1 + assert 'output1' in result.keys() + assert result['output1'].shape == (8, 3) + assert np.all(result['output1'] == np.zeros([8, 3])) + + def test_two_dims_in_wildcard_with_basic_dim_and_extra_basic_dim_in_input(self): + output_properties = { + 'output1': { + 'dims': ['*', 'dim3'], + 'units': 'm', + } + } + input_properties = { + 'input1': { + 'dims': ['*', 'dim3', 'dim4'], + 'units': 's^-1', + } + } + input_state = { + 'input1': DataArray( + np.zeros([4, 3, 2, 5]), + dims=['dim1', 'dim2', 'dim3', 'dim4'], + attrs={'units': 's^-1'} + ) + } + + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert len(result.keys()) == 1 + assert 'output1' in result.keys() + assert result['output1'].shape == (12, 2) + assert np.all(result['output1'] == np.zeros([12, 2])) From 8db2103630093c85cb006a4bd8dcf7c756a368d1 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 9 Apr 2018 11:21:09 -0700 Subject: [PATCH 48/98] removed print statement --- sympl/_core/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sympl/_core/constants.py b/sympl/_core/constants.py index 040f3f0..809fd3d 100644 --- a/sympl/_core/constants.py +++ b/sympl/_core/constants.py @@ -22,7 +22,6 @@ def _repr(self, sphinx=False): if sphinx: return_string += '\n' return_string += '\n' - print(set(self.keys()).difference(printed_names)) if len(set(self.keys()).difference(printed_names)) > 0: return_string += 'User Defined\n' for name in self.keys(): From 5896f25d986b32f2282354809a7e796ea87ade0e Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Apr 2018 13:14:13 -0700 Subject: [PATCH 49/98] Added reference_air_temperature constant --- HISTORY.rst | 1 + sympl/_core/constants.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 04c4d2b..a7d8495 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -42,6 +42,7 @@ Latest * Fixed a bug where ABCMeta was not being used in Python 3. * Added initialize_numpy_arrays_with_properties which creates zero arrays for an output properties dictionary. +* Added reference_air_temperature constant Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/_core/constants.py b/sympl/_core/constants.py index 809fd3d..4a16c9d 100644 --- a/sympl/_core/constants.py +++ b/sympl/_core/constants.py @@ -107,6 +107,7 @@ def __getitem__(self, item): 'heat_capacity_of_solid_water_as_snow': DataArray(2108., attrs={'units': 'J kg^-1 K^-1'}), 'thermal_conductivity_of_solid_water_as_snow': DataArray(0.2, attrs={'units': 'W m^-1 K^-1'}), 'reference_air_pressure': DataArray(1.0132e5, attrs={'units': 'Pa'}), + 'reference_air_temperature': DataArray(300., attrs={'units': 'degK'}), 'thermal_conductivity_of_dry_air': DataArray(0.026, attrs={'units': 'W m^-1 K^-1'}), 'gas_constant_of_dry_air': DataArray(287., attrs={'units': 'J kg^-1 K^-1'}), 'gas_constant_of_water_vapor': DataArray(461.5, attrs={'units': 'J kg^-1 K^-1'}), @@ -169,7 +170,9 @@ def __getitem__(self, item): 'heat_capacity_of_dry_air_at_constant_pressure', 'gas_constant_of_dry_air', 'thermal_conductivity_of_dry_air', - 'reference_air_pressure'], + 'reference_air_pressure', + 'reference_air_temperature', + ], 'stellar': [ 'stellar_irradiance', From 3909a5990622548c5e6f34a982e7be76192c6789 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Apr 2018 13:14:57 -0700 Subject: [PATCH 50/98] Added additional test that timesteppers don't modify input state --- tests/test_timestepping.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index e310ea9..ecfacb7 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -371,6 +371,44 @@ def test_given_tendency_not_modified_with_two_components(self): assert np.all(tendency_output_1['tend1'] == 5.) assert np.all(tendency_output_2['tend1'] == 5.) + def test_input_state_not_modified_with_two_components(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'tend1': { + 'dims': ['dim1'], + 'units': 'm/s', + } + } + diagnostic_output = {} + tendency_output_1 = { + 'tend1': np.ones([10]) * 5. + } + prognostic1 = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output_1 + ) + tendency_output_2 = { + 'tend1': np.ones([10]) * 5. + } + prognostic2 = MockPrognostic( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output_2 + ) + stepper = self.timestepper_class( + prognostic1, prognostic2, tendencies_in_diagnostics=True) + state = { + 'time': timedelta(0), + 'tend1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'}, + ) + } + _, _ = stepper(state, timedelta(seconds=5)) + assert state['tend1'].attrs['units'] == 'm' + assert np.all(state['tend1'].values == 1.) + def test_tendencies_in_diagnostics_no_tendency(self): input_properties = {} diagnostic_properties = {} From 957d50ffb8297e9b1db34e8c87fbf4e0c39f6a61 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Apr 2018 13:16:20 -0700 Subject: [PATCH 51/98] Base components now warn if output property units are incompatible with input property units --- HISTORY.rst | 2 ++ sympl/_core/base_components.py | 59 ++++++++++++++++++++++++++++++++++ tests/test_base_components.py | 58 +++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index a7d8495..a604ed4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -43,6 +43,8 @@ Latest * Added initialize_numpy_arrays_with_properties which creates zero arrays for an output properties dictionary. * Added reference_air_temperature constant +* Base classes now emit warnings when output property units conflict with input + property units (which probably indicates that they're wrong). Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 3e4d629..d3f94c8 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -5,6 +5,8 @@ InvalidPropertyDictError, ComponentExtraOutputError, ComponentMissingOutputError, InvalidStateError) from six import add_metaclass +import warnings +from .units import units_are_compatible try: from inspect import getfullargspec as getargspec except ImportError: @@ -125,6 +127,36 @@ def check_inputs(self, state): raise InvalidStateError('Missing input quantity {}'.format(key)) +def get_name_with_incompatible_units(properties1, properties2): + """ + If there are any keys shared by the two properties + dictionaries which indicate units that are incompatible with one another, + this returns such a key. Otherwise returns None. + """ + for name in set(properties1.keys()).intersection(properties2.keys()): + if not units_are_compatible( + properties1[name]['units'], properties2[name]['units']): + return name + return None + + +def get_tendency_name_with_incompatible_units(input_properties, tendency_properties): + """ + Returns False if there are any keys shared by the two properties + dictionaries which indicate units that are incompatible with one another, + and True otherwise (if there are no conflicting unit specifications). + """ + for name in set(input_properties.keys()).intersection(tendency_properties.keys()): + if input_properties[name]['units'] == '': + expected_tendency_units = 's^-1' + else: + expected_tendency_units = input_properties[name]['units'] + ' s^-1' + if not units_are_compatible( + expected_tendency_units, tendency_properties[name]['units']): + return name + return None + + class TendencyChecker(object): def __init__(self, component): @@ -138,6 +170,15 @@ def __init__(self, component): 'Tendency properties do not have dims defined for {}'.format(name) ) check_overlapping_aliases(self.component.tendency_properties, 'tendency') + incompatible_name = get_tendency_name_with_incompatible_units( + self.component.input_properties, self.component.tendency_properties) + if incompatible_name is not None: + warnings.warn( + 'Component of type {} has input {} with tendency units {} that ' + 'are incompatible with input units {}'.format( + type(self.component), incompatible_name, + self.component.tendency_properties[incompatible_name]['units'], + self.component.input_properties[incompatible_name]['units'])) super(TendencyChecker, self).__init__() @property @@ -193,6 +234,15 @@ def __init__(self, component): raise InvalidPropertyDictError( 'Diagnostic properties do not have dims defined for {}'.format(name) ) + incompatible_name = get_name_with_incompatible_units( + self.component.input_properties, self.component.diagnostic_properties) + if incompatible_name is not None: + warnings.warn( + 'Component of type {} has input {} with diagnostic units {} that ' + 'are incompatible with input units {}'.format( + type(self.component), incompatible_name, + self.component.diagnostic_properties[incompatible_name]['units'], + self.component.input_properties[incompatible_name]['units'])) check_overlapping_aliases(component.diagnostic_properties, 'diagnostic') @property @@ -253,6 +303,15 @@ def __init__(self, component): 'Output properties do not have dims defined for {}'.format(name) ) check_overlapping_aliases(self.component.output_properties, 'output') + incompatible_name = get_name_with_incompatible_units( + self.component.input_properties, self.component.output_properties) + if incompatible_name is not None: + warnings.warn( + 'Component of type {} has input {} with output units {} that ' + 'are incompatible with input units {}'.format( + type(self.component), incompatible_name, + self.component.output_properties[incompatible_name]['units'], + self.component.input_properties[incompatible_name]['units'])) super(OutputChecker, self).__init__() @property diff --git a/tests/test_base_components.py b/tests/test_base_components.py index 5a628a7..af9d62f 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -8,6 +8,7 @@ ComponentMissingOutputError, ComponentExtraOutputError, InvalidStateError ) +import warnings def same_list(list1, list2): return (len(list1) == len(list2) and all( @@ -425,6 +426,25 @@ def test_diagnostic_requires_units(self): with self.assertRaises(InvalidPropertyDictError): self.get_component(diagnostic_properties=diagnostic_properties) + def test_diagnostic_warns_when_units_incompatible_with_input(self): + input_properties = { + 'diag1': {'units': 'km', 'dims': ['dim1', 'dim2']} + } + diagnostic_properties = { + 'diag1': {'units': 'seconds', 'dims': ['dim1', 'dim2']} + } + with warnings.catch_warnings(record=True) as w: + self.get_component( + input_properties=input_properties, + diagnostic_properties=diagnostic_properties + ) + assert len(w) == 1 + assert issubclass(w[-1].category, UserWarning) + assert 'units' in str(w[-1].message) + assert 'diag1' in str(w[-1].message) + assert 'seconds' in str(w[-1].message) + assert 'km' in str(w[-1].message) + def test_diagnostic_requires_correct_number_of_dims(self): input_properties = { 'input1': {'units': 'm', 'dims': ['dim1', 'dim2']} @@ -728,6 +748,25 @@ def array_call(self): instance = MyPrognostic() assert isinstance(instance, Prognostic) + def test_tendency_warns_when_units_incompatible_with_input(self): + input_properties = { + 'input1': {'units': 'km', 'dims': ['dim1', 'dim2']} + } + tendency_properties = { + 'input1': {'units': 'degK/s', 'dims': ['dim1', 'dim2']} + } + with warnings.catch_warnings(record=True) as w: + self.get_component( + input_properties=input_properties, + tendency_properties=tendency_properties + ) + assert len(w) == 1 + assert issubclass(w[-1].category, UserWarning) + assert 'units' in str(w[-1].message) + assert 'input1' in str(w[-1].message) + assert 'degK/s' in str(w[-1].message) + assert 'km' in str(w[-1].message) + def test_two_components_are_not_instances_of_each_other(self): class MyPrognostic1(Prognostic): input_properties = {} @@ -1480,6 +1519,25 @@ def array_call(self, state, timestep): instance = MyImplicit() assert isinstance(instance, Implicit) + def test_output_warns_when_units_incompatible_with_input(self): + input_properties = { + 'input1': {'units': 'km', 'dims': ['dim1', 'dim2']} + } + output_properties = { + 'input1': {'units': 'degK', 'dims': ['dim1', 'dim2']} + } + with warnings.catch_warnings(record=True) as w: + self.get_component( + input_properties=input_properties, + output_properties=output_properties, + ) + assert len(w) == 1 + assert issubclass(w[-1].category, UserWarning) + assert 'units' in str(w[-1].message) + assert 'input1' in str(w[-1].message) + assert 'degK' in str(w[-1].message) + assert 'km' in str(w[-1].message) + def test_two_components_are_not_instances_of_each_other(self): class MyImplicit1(Implicit): input_properties = {} From f504b7b5129b98c3a30ab56c2c8f680a69260203 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Apr 2018 13:28:29 -0700 Subject: [PATCH 52/98] Switched from netcdftime to cftime (rename of package) for datetime handling --- sympl/_core/time.py | 20 ++++++++++---------- tox.ini | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sympl/_core/time.py b/sympl/_core/time.py index ee1ff43..f0401d8 100644 --- a/sympl/_core/time.py +++ b/sympl/_core/time.py @@ -1,13 +1,13 @@ from datetime import datetime as real_datetime, timedelta from .exceptions import DependencyError try: - import netcdftime as nt - if not all(hasattr(nt, attr) for attr in [ + import cftime as ct + if not all(hasattr(ct, attr) for attr in [ 'DatetimeNoLeap', 'DatetimeProlepticGregorian', 'DatetimeAllLeap', 'Datetime360Day', 'DatetimeJulian', 'DatetimeGregorian']): - nt = None + ct = None except ImportError: - nt = None + ct = None def datetime( @@ -51,20 +51,20 @@ def datetime( return real_datetime(tzinfo=tzinfo, **kwargs) elif tzinfo is not None: raise ValueError('netcdftime does not support timezone-aware datetimes') - elif nt is None: + elif ct is None: raise DependencyError( "Calendars other than 'proleptic_gregorian' require the netcdftime " "package, which is not installed.") elif calendar.lower() in ('all_leap', '366_day'): - return nt.DatetimeAllLeap(**kwargs) + return ct.DatetimeAllLeap(**kwargs) elif calendar.lower() in ('no_leap', 'noleap', '365_day'): - return nt.DatetimeNoLeap(**kwargs) + return ct.DatetimeNoLeap(**kwargs) elif calendar.lower() == '360_day': - return nt.Datetime360Day(**kwargs) + return ct.Datetime360Day(**kwargs) elif calendar.lower() == 'julian': - return nt.DatetimeJulian(**kwargs) + return ct.DatetimeJulian(**kwargs) elif calendar.lower() == 'gregorian': - return nt.DatetimeGregorian(**kwargs) + return ct.DatetimeGregorian(**kwargs) __all__ = (datetime, timedelta) diff --git a/tox.ini b/tox.ini index 1d140b5..04f9fa6 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ deps = -r{toxinidir}/requirements_dev.txt commands = pip install -U pip - pip install netcdftime + pip install cftime py.test --basetemp={envtmpdir} -v [testenv:cov] @@ -26,7 +26,7 @@ commands = passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH usedevelop = true commands = - pip install codecov netcdftime + pip install codecov cftime py.test --cov=sympl codecov --token=970a49ea-e6db-4abb-a7e2-909b139e0ce1 deps = From 799053f7dadeab76662e179cfe0dd38a690803bf Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Apr 2018 13:30:01 -0700 Subject: [PATCH 53/98] Updated tests to use cftime instead of netcdftime --- tests/test_time.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/tests/test_time.py b/tests/test_time.py index 71fcf6a..d4feb57 100644 --- a/tests/test_time.py +++ b/tests/test_time.py @@ -3,13 +3,9 @@ import unittest import pytz try: - import netcdftime as nt - if not all(hasattr(nt, attr) for attr in [ - 'DatetimeNoLeap', 'DatetimeProlepticGregorian', 'DatetimeAllLeap', - 'Datetime360Day', 'DatetimeJulian', 'DatetimeGregorian']): - nt = None + import cftime as ct except ImportError: - nt = None + ct = None netcdftime_not_installed = 'netcdftime module is not installed' @@ -74,24 +70,24 @@ def testTimezoneAwareDatetimeHasCorrectValues(self): assert tz_dt.microsecond == 0 -@unittest.skipIf(nt is None, netcdftime_not_installed) +@unittest.skipIf(ct is None, netcdftime_not_installed) class NoLeapTests(unittest.TestCase, DatetimeBase): calendar = 'no_leap' @property def dt_class(self): - return nt.DatetimeNoLeap + return ct.DatetimeNoLeap -@unittest.skipIf(nt is None, netcdftime_not_installed) +@unittest.skipIf(ct is None, netcdftime_not_installed) class Datetime365DayTests(unittest.TestCase, DatetimeBase): calendar = '365_day' @property def dt_class(self): - return nt.DatetimeNoLeap + return ct.DatetimeNoLeap def test_incrementing_years_using_days(self): dt = datetime(1900, 1, 1, calendar=self.calendar) @@ -110,22 +106,22 @@ def test_decrementing_years_using_days(self): assert dt_1850.hour == 0 -@unittest.skipIf(nt is None, netcdftime_not_installed) +@unittest.skipIf(ct is None, netcdftime_not_installed) class AllLeapTests(unittest.TestCase, DatetimeBase): calendar = 'all_leap' @property def dt_class(self): - return nt.DatetimeAllLeap + return ct.DatetimeAllLeap -@unittest.skipIf(nt is None, netcdftime_not_installed) +@unittest.skipIf(ct is None, netcdftime_not_installed) class Datetime366DayTests(unittest.TestCase, DatetimeBase): calendar = '366_day' @property def dt_class(self): - return nt.DatetimeAllLeap + return ct.DatetimeAllLeap def test_incrementing_years_using_days(self): dt = datetime(1900, 1, 1, calendar=self.calendar) @@ -144,13 +140,13 @@ def test_decrementing_years_using_days(self): assert dt_1850.hour == 0 -@unittest.skipIf(nt is None, netcdftime_not_installed) +@unittest.skipIf(ct is None, netcdftime_not_installed) class Datetime360DayTests(unittest.TestCase, DatetimeBase): calendar = '360_day' @property def dt_class(self): - return nt.Datetime360Day + return ct.Datetime360Day def test_incrementing_years_using_days(self): dt = datetime(1900, 1, 1, calendar=self.calendar) @@ -169,19 +165,19 @@ def test_decrementing_years_using_days(self): assert dt_1850.hour == 0 -@unittest.skipIf(nt is None, netcdftime_not_installed) +@unittest.skipIf(ct is None, netcdftime_not_installed) class JulianTests(unittest.TestCase, DatetimeBase): calendar = 'julian' @property def dt_class(self): - return nt.DatetimeJulian + return ct.DatetimeJulian -unittest.skipIf(nt is None, netcdftime_not_installed) +unittest.skipIf(ct is None, netcdftime_not_installed) class GregorianTests(unittest.TestCase, DatetimeBase): calendar = 'gregorian' @property def dt_class(self): - return nt.DatetimeGregorian + return ct.DatetimeGregorian From 5c8bda7f1a4ab8bfc35242c7570aa9cb575e3fd5 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Apr 2018 13:30:56 -0700 Subject: [PATCH 54/98] Corrected message for when cftime is not installed --- tests/test_time.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_time.py b/tests/test_time.py index d4feb57..c6a9712 100644 --- a/tests/test_time.py +++ b/tests/test_time.py @@ -7,7 +7,7 @@ except ImportError: ct = None -netcdftime_not_installed = 'netcdftime module is not installed' +netcdftime_not_installed = 'cftime module is not installed' class DatetimeBase(object): From 9748df8fcfe86d319bedd4ad96cc9e103240bfc1 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Apr 2018 13:50:25 -0700 Subject: [PATCH 55/98] Added test for temperature unit conversion in inputs, fixed related bug --- HISTORY.rst | 2 ++ sympl/_core/units.py | 2 +- tests/test_base_components.py | 23 +++++++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index a604ed4..c5fdffa 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -45,6 +45,8 @@ Latest * Added reference_air_temperature constant * Base classes now emit warnings when output property units conflict with input property units (which probably indicates that they're wrong). +* Fixed bug where degrees Celcius or Fahrenheit could not be used as units on inputs + because it would lead to an error Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/_core/units.py b/sympl/_core/units.py index 3c4d9fc..29358ab 100644 --- a/sympl/_core/units.py +++ b/sympl/_core/units.py @@ -50,7 +50,7 @@ def data_array_to_units(value, units): 'Cannot retrieve units from type {}'.format(type(value))) elif unit_registry(value.attrs['units']) != unit_registry(units): attrs = value.attrs.copy() - value = (unit_registry(value.attrs['units'])*value).to(units).magnitude + value = unit_registry.Quantity(value, value.attrs['units']).to(units).magnitude attrs['units'] = units value.attrs = attrs return value diff --git a/tests/test_base_components.py b/tests/test_base_components.py index af9d62f..d0d4375 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -366,6 +366,29 @@ def test_input_converts_units(self): assert isinstance(component.state_given['input1'], np.ndarray) assert np.all(component.state_given['input1'] == np.ones([10])*1000.) + def test_input_converts_temperature_units(self): + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 'degK' + } + } + component = self.get_component(input_properties=input_properties) + state = { + 'time': timedelta(0), + 'input1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'degC'} + ) + } + self.call_component(component, state) + assert len(component.state_given) == 2 + assert 'time' in component.state_given.keys() + assert 'input1' in component.state_given.keys() + assert isinstance(component.state_given['input1'], np.ndarray) + assert np.all(component.state_given['input1'] == np.ones([10])*274.15) + def test_input_collects_one_dimension(self): input_properties = { 'input1': { From 06c13bf88ff03144cea39b8a59b22af605120fae Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Apr 2018 16:33:03 -0700 Subject: [PATCH 56/98] loosened dev requirements --- requirements_dev.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 1727b5f..7f17745 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -2,15 +2,15 @@ pip>=8.1.2 bumpversion==0.5.3 wheel==0.29.0 watchdog==0.8.3 -flake8==2.6.0 +flake8>=2.6.0 tox==2.3.1 coverage==4.1 Sphinx==1.4.8 -cryptography==1.4 -PyYAML==3.11 -pytest==2.9.2 -pytest-cov==2.4.0 -mock==2.0.0 +cryptography>=1.4 +PyYAML>=3.11 +pytest>=2.9.2 +pytest-cov>=2.4.0 +mock>=2.0.0 xarray>=0.9.3 six pint==0.7.0 From 7ab79c2f2d117a6c21346780b624ae6fedef4c2c Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Apr 2018 16:37:08 -0700 Subject: [PATCH 57/98] Turned all dev requirements from == to >= --- requirements_dev.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 7f17745..1e76fbe 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,11 +1,11 @@ pip>=8.1.2 -bumpversion==0.5.3 -wheel==0.29.0 -watchdog==0.8.3 +bumpversion>=0.5.3 +wheel>=0.29.0 +watchdog>=0.8.3 flake8>=2.6.0 -tox==2.3.1 -coverage==4.1 -Sphinx==1.4.8 +tox>=2.3.1 +coverage>=4.1 +Sphinx>=1.4.8 cryptography>=1.4 PyYAML>=3.11 pytest>=2.9.2 @@ -13,7 +13,7 @@ pytest-cov>=2.4.0 mock>=2.0.0 xarray>=0.9.3 six -pint==0.7.0 +pint>=0.7.0 scipy numpy>=0.10 coveralls From 629bb82f49f3fd2452360a70b1a540184303bb27 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Apr 2018 16:40:41 -0700 Subject: [PATCH 58/98] Made pytest dev requirement specific --- requirements_dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 1e76fbe..b41a541 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -8,7 +8,7 @@ coverage>=4.1 Sphinx>=1.4.8 cryptography>=1.4 PyYAML>=3.11 -pytest>=2.9.2 +pytest==2.9.2 pytest-cov>=2.4.0 mock>=2.0.0 xarray>=0.9.3 From 50a6933fcd7bab943dcec75579d19c340e2ad4ae Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Apr 2018 16:41:58 -0700 Subject: [PATCH 59/98] Undo last commit --- requirements_dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index b41a541..1e76fbe 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -8,7 +8,7 @@ coverage>=4.1 Sphinx>=1.4.8 cryptography>=1.4 PyYAML>=3.11 -pytest==2.9.2 +pytest>=2.9.2 pytest-cov>=2.4.0 mock>=2.0.0 xarray>=0.9.3 From 3f6f54ab80dc6662c1461a4496cd35afe7ef3977 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 16 May 2018 14:07:35 -0700 Subject: [PATCH 60/98] Changed warnings on invalid property dict units into InvalidPropertyDictError --- HISTORY.rst | 4 ++-- sympl/_core/base_components.py | 7 +++--- tests/test_base_components.py | 43 ++++++++++------------------------ 3 files changed, 18 insertions(+), 36 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c5fdffa..508e4cb 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -43,8 +43,6 @@ Latest * Added initialize_numpy_arrays_with_properties which creates zero arrays for an output properties dictionary. * Added reference_air_temperature constant -* Base classes now emit warnings when output property units conflict with input - property units (which probably indicates that they're wrong). * Fixed bug where degrees Celcius or Fahrenheit could not be used as units on inputs because it would lead to an error @@ -55,6 +53,8 @@ Breaking changes In order to get these, you should use e.g. input_properties.keys() * properties dictionaries are now abstract methods, so subclasses must define them. Previously they defaulted to empty dictionaries. +* Base classes now raise InvalidPropertyDictError when output property units conflict with input + property units (which probably indicates that they're wrong). * Components should now be written using a new array_call method rather than __call__. __call__ will automatically unwrap DataArrays to numpy arrays to be passed into array_call based on the component's properties dictionaries, and re-wrap to diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index d3f94c8..90f396a 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -5,7 +5,6 @@ InvalidPropertyDictError, ComponentExtraOutputError, ComponentMissingOutputError, InvalidStateError) from six import add_metaclass -import warnings from .units import units_are_compatible try: from inspect import getfullargspec as getargspec @@ -173,7 +172,7 @@ def __init__(self, component): incompatible_name = get_tendency_name_with_incompatible_units( self.component.input_properties, self.component.tendency_properties) if incompatible_name is not None: - warnings.warn( + raise InvalidPropertyDictError( 'Component of type {} has input {} with tendency units {} that ' 'are incompatible with input units {}'.format( type(self.component), incompatible_name, @@ -237,7 +236,7 @@ def __init__(self, component): incompatible_name = get_name_with_incompatible_units( self.component.input_properties, self.component.diagnostic_properties) if incompatible_name is not None: - warnings.warn( + raise InvalidPropertyDictError( 'Component of type {} has input {} with diagnostic units {} that ' 'are incompatible with input units {}'.format( type(self.component), incompatible_name, @@ -306,7 +305,7 @@ def __init__(self, component): incompatible_name = get_name_with_incompatible_units( self.component.input_properties, self.component.output_properties) if incompatible_name is not None: - warnings.warn( + raise InvalidPropertyDictError( 'Component of type {} has input {} with output units {} that ' 'are incompatible with input units {}'.format( type(self.component), incompatible_name, diff --git a/tests/test_base_components.py b/tests/test_base_components.py index d0d4375..7c37b8f 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -8,7 +8,6 @@ ComponentMissingOutputError, ComponentExtraOutputError, InvalidStateError ) -import warnings def same_list(list1, list2): return (len(list1) == len(list2) and all( @@ -449,24 +448,18 @@ def test_diagnostic_requires_units(self): with self.assertRaises(InvalidPropertyDictError): self.get_component(diagnostic_properties=diagnostic_properties) - def test_diagnostic_warns_when_units_incompatible_with_input(self): + def test_diagnostic_raises_when_units_incompatible_with_input(self): input_properties = { 'diag1': {'units': 'km', 'dims': ['dim1', 'dim2']} } diagnostic_properties = { 'diag1': {'units': 'seconds', 'dims': ['dim1', 'dim2']} } - with warnings.catch_warnings(record=True) as w: + with self.assertRaises(InvalidPropertyDictError): self.get_component( input_properties=input_properties, diagnostic_properties=diagnostic_properties ) - assert len(w) == 1 - assert issubclass(w[-1].category, UserWarning) - assert 'units' in str(w[-1].message) - assert 'diag1' in str(w[-1].message) - assert 'seconds' in str(w[-1].message) - assert 'km' in str(w[-1].message) def test_diagnostic_requires_correct_number_of_dims(self): input_properties = { @@ -555,6 +548,7 @@ def test_diagnostics_no_transformations(self): assert len(diagnostics['output1'].dims) == 1 assert 'dim1' in diagnostics['output1'].dims assert 'units' in diagnostics['output1'].attrs + assert len(diagnostics['output1'].attrs) == 1 assert diagnostics['output1'].attrs['units'] == 'm' assert np.all(diagnostics['output1'].values == np.ones([10])) @@ -771,24 +765,18 @@ def array_call(self): instance = MyPrognostic() assert isinstance(instance, Prognostic) - def test_tendency_warns_when_units_incompatible_with_input(self): + def test_tendency_raises_when_units_incompatible_with_input(self): input_properties = { 'input1': {'units': 'km', 'dims': ['dim1', 'dim2']} } tendency_properties = { 'input1': {'units': 'degK/s', 'dims': ['dim1', 'dim2']} } - with warnings.catch_warnings(record=True) as w: + with self.assertRaises(InvalidPropertyDictError): self.get_component( input_properties=input_properties, tendency_properties=tendency_properties ) - assert len(w) == 1 - assert issubclass(w[-1].category, UserWarning) - assert 'units' in str(w[-1].message) - assert 'input1' in str(w[-1].message) - assert 'degK/s' in str(w[-1].message) - assert 'km' in str(w[-1].message) def test_two_components_are_not_instances_of_each_other(self): class MyPrognostic1(Prognostic): @@ -865,7 +853,7 @@ def test_tendency_requires_dims(self): def test_tendency_uses_base_dims(self): input_properties = {'diag1': {'dims': ['dim1'], 'units': 'm'}} diagnostic_properties = {} - tendency_properties = {'diag1': {'units': 'm'}} + tendency_properties = {'diag1': {'units': 'm/s'}} diagnostic_output = {} tendency_output = {} self.component_class( @@ -1045,6 +1033,7 @@ def test_tendencies_no_transformations(self): assert len(tendencies['output1'].dims) == 1 assert 'dim1' in tendencies['output1'].dims assert 'units' in tendencies['output1'].attrs + assert len(tendencies['output1'].attrs) == 1 assert tendencies['output1'].attrs['units'] == 'm/s' assert np.all(tendencies['output1'].values == np.ones([10])) @@ -1542,24 +1531,18 @@ def array_call(self, state, timestep): instance = MyImplicit() assert isinstance(instance, Implicit) - def test_output_warns_when_units_incompatible_with_input(self): + def test_output_raises_when_units_incompatible_with_input(self): input_properties = { 'input1': {'units': 'km', 'dims': ['dim1', 'dim2']} } output_properties = { 'input1': {'units': 'degK', 'dims': ['dim1', 'dim2']} } - with warnings.catch_warnings(record=True) as w: + with self.assertRaises(InvalidPropertyDictError): self.get_component( input_properties=input_properties, output_properties=output_properties, ) - assert len(w) == 1 - assert issubclass(w[-1].category, UserWarning) - assert 'units' in str(w[-1].message) - assert 'input1' in str(w[-1].message) - assert 'degK' in str(w[-1].message) - assert 'km' in str(w[-1].message) def test_two_components_are_not_instances_of_each_other(self): class MyImplicit1(Implicit): @@ -1804,7 +1787,7 @@ def test_output_with_alias_from_input(self): output_properties = { 'output1': { 'dims': ['dim1'], - 'units': 'm/s', + 'units': 'm', } } diagnostic_output = {} @@ -1830,7 +1813,7 @@ def test_output_with_alias_from_input(self): assert len(output['output1'].dims) == 1 assert 'dim1' in output['output1'].dims assert 'units' in output['output1'].attrs - assert output['output1'].attrs['units'] == 'm/s' + assert output['output1'].attrs['units'] == 'm' assert np.all(output['output1'].values == np.ones([10])) def test_output_with_dims_from_input(self): @@ -1843,7 +1826,7 @@ def test_output_with_dims_from_input(self): diagnostic_properties = {} output_properties = { 'output1': { - 'units': 'm/s', + 'units': 'm', } } diagnostic_output = {} @@ -1869,7 +1852,7 @@ def test_output_with_dims_from_input(self): assert len(output['output1'].dims) == 1 assert 'dim1' in output['output1'].dims assert 'units' in output['output1'].attrs - assert output['output1'].attrs['units'] == 'm/s' + assert output['output1'].attrs['units'] == 'm' assert np.all(output['output1'].values == np.ones([10])) def test_tendencies_in_diagnostics_no_tendency(self): From 054a434192367321f82211c459722149efc7f25a Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 16 May 2018 15:28:12 -0700 Subject: [PATCH 61/98] Fixed units on some mock components --- tests/test_composite.py | 32 ++++++++++++++++---------------- tests/test_wrapper.py | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test_composite.py b/tests/test_composite.py index b320a7f..20ce596 100644 --- a/tests/test_composite.py +++ b/tests/test_composite.py @@ -615,7 +615,7 @@ def test_prognostic_composite_implicit_dims(): input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', } }, diagnostic_properties={}, @@ -643,7 +643,7 @@ def test_two_prognostic_composite_implicit_dims(): input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', } }, diagnostic_properties={}, @@ -659,7 +659,7 @@ def test_two_prognostic_composite_implicit_dims(): input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', } }, diagnostic_properties={}, @@ -687,7 +687,7 @@ def test_prognostic_composite_explicit_dims(): input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', } }, diagnostic_properties={}, @@ -711,7 +711,7 @@ def test_two_prognostic_composite_explicit_dims(): input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', } }, diagnostic_properties={}, @@ -728,7 +728,7 @@ def test_two_prognostic_composite_explicit_dims(): input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', } }, diagnostic_properties={}, @@ -752,7 +752,7 @@ def test_two_prognostic_composite_explicit_and_implicit_dims(): input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', } }, diagnostic_properties={}, @@ -769,7 +769,7 @@ def test_two_prognostic_composite_explicit_and_implicit_dims(): input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', } }, diagnostic_properties={}, @@ -792,7 +792,7 @@ def test_prognostic_composite_explicit_dims_not_in_input(): input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', } }, diagnostic_properties={}, @@ -816,11 +816,11 @@ def test_two_prognostic_composite_incompatible_dims(): input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', }, 'input2': { 'dims': ['dims3', 'dims1'], - 'units': 'degK / day' + 'units': 'degK' } }, diagnostic_properties={}, @@ -837,7 +837,7 @@ def test_two_prognostic_composite_incompatible_dims(): input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', }, 'input2': { 'dims': ['dims3', 'dims1'], @@ -867,11 +867,11 @@ def test_two_prognostic_composite_compatible_dims(): input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', }, 'input2': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day' + 'units': 'degK' } }, diagnostic_properties={}, @@ -888,11 +888,11 @@ def test_two_prognostic_composite_compatible_dims(): input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day', + 'units': 'degK', }, 'input2': { 'dims': ['dims1', 'dims2'], - 'units': 'degK / day' + 'units': 'degK' } }, diagnostic_properties={}, diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index 200c021..9cef4b8 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -624,7 +624,7 @@ def test_tendency_no_scaling_when_input_scaled(self): self.tendency_properties = { 'diag1': { 'dims': ['dim1'], - 'units': 'm', + 'units': 'm/s', } } self.tendency_output = { From fb4f0df255d7be7fb71fca1b1cb292aea991ffb6 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 24 May 2018 12:33:37 -0700 Subject: [PATCH 62/98] Added spyder project files to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 60200d7..95a5c0f 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,6 @@ target/ # pyenv python configuration file .python-version + +# Spyder +.spyderproject From 653d7926509fbb0f21c01453e05a13d722f5b6ab Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 13 Jun 2018 17:12:59 -0700 Subject: [PATCH 63/98] Added combine_component_properties as a public function --- HISTORY.rst | 7 +- sympl/__init__.py | 3 +- sympl/_core/base_components.py | 20 ++-- sympl/_core/state.py | 43 ++++--- tests/test_get_restore_numpy_array.py | 163 ++++++++++++++++++-------- 5 files changed, 160 insertions(+), 76 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 508e4cb..b322613 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -28,7 +28,7 @@ Latest dictionaries, raising ComponentMissingOutputError or ComponentExtraOutputError respectively if outputs do not match. * Added a priority order of property types for determining which aliases are - returned by get_component_aliases + returned by get_component_aliases. * Fixed a bug where TimeStepper objects would modify the arrays passed to them by Prognostic objects, leading to unexpected value changes. * Fixed a bug where constants were missing from the string returned by @@ -42,9 +42,10 @@ Latest * Fixed a bug where ABCMeta was not being used in Python 3. * Added initialize_numpy_arrays_with_properties which creates zero arrays for an output properties dictionary. -* Added reference_air_temperature constant +* Added reference_air_temperature constant. * Fixed bug where degrees Celcius or Fahrenheit could not be used as units on inputs - because it would lead to an error + because it would lead to an error. +* Added combine_component_properties as a public function. Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/__init__.py b/sympl/__init__.py index 7c7cc7f..ad19273 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -18,7 +18,8 @@ ensure_no_shared_keys, get_numpy_array, jit, restore_dimensions, - get_component_aliases) + get_component_aliases, + combine_component_properties) from ._core.state import ( get_numpy_arrays_with_properties, restore_data_arrays_with_properties, diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 90f396a..eff7993 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -526,7 +526,8 @@ def __call__(self, state, timestep): Args ---- state : dict - A model state dictionary. + A model state dictionary satisfying the input_properties of this + object. timestep : timedelta The amount of time to step forward. @@ -583,7 +584,7 @@ def array_call(self, state, timestep): ---- state : dict A numpy array state dictionary. Instead of data arrays, should - include numpy arrays that satisfy the input properties of this + include numpy arrays that satisfy the input_properties of this object. timestep : timedelta The amount of time to step forward. @@ -734,7 +735,8 @@ def __call__(self, state): Args ---- state : dict - A model state dictionary. + A model state dictionary satisfying the input_properties of this + object. Returns ------- @@ -787,7 +789,7 @@ def array_call(self, state): ---- state : dict A model state dictionary. Instead of data arrays, should - include numpy arrays that satisfy the input properties of this + include numpy arrays that satisfy the input_properties of this object. Returns @@ -937,7 +939,8 @@ def __call__(self, state, timestep): Args ---- state : dict - A model state dictionary. + A model state dictionary satisfying the input_properties of this + object. timestep : timedelta The time over which the model is being stepped. @@ -993,7 +996,7 @@ def array_call(self, state, timestep): ---- state : dict A model state dictionary. Instead of data arrays, should - include numpy arrays that satisfy the input properties of this + include numpy arrays that satisfy the input_properties of this object. timestep : timedelta The time over which the model is being stepped. @@ -1087,7 +1090,8 @@ def __call__(self, state): Args ---- state : dict - A model state dictionary. + A model state dictionary satisfying the input_properties of this + object. Returns ------- @@ -1123,7 +1127,7 @@ def array_call(self, state): ---- state : dict A model state dictionary. Instead of data arrays, should - include numpy arrays that satisfy the input properties of this + include numpy arrays that satisfy the input_properties of this object. Returns diff --git a/sympl/_core/state.py b/sympl/_core/state.py index 71ab05a..d8486a2 100644 --- a/sympl/_core/state.py +++ b/sympl/_core/state.py @@ -146,7 +146,7 @@ def get_numpy_array(data_array, out_dims, dim_lengths): def initialize_numpy_arrays_with_properties( - output_properties, input_state, input_properties, dtype=np.float64): + output_properties, raw_input_state, input_properties, dtype=np.float64): """ Parameters ---------- @@ -154,9 +154,9 @@ def initialize_numpy_arrays_with_properties( A dictionary whose keys are quantity names and values are dictionaries with properties for those quantities. The property "dims" must be present for each quantity not also present in input_properties. - input_state : dict - A state dictionary that was used as input to a component for which - DataArrays are being restored. + raw_input_state : dict + A state dictionary of numpy arrays that was used as input to a component + for which return arrays are being generated. input_properties : dict A dictionary whose keys are quantity names and values are dictionaries with input properties for those quantities. The property "dims" must be @@ -177,27 +177,33 @@ def initialize_numpy_arrays_with_properties( property, but the arrays for the two properties have incompatible shapes. """ - wildcard_names, dim_lengths = get_wildcard_matches_and_dim_lengths( - input_state, input_properties) + dim_lengths = get_dim_lengths_from_raw_input(raw_input_state, input_properties) dims_from_out_properties = extract_output_dims_properties( output_properties, input_properties, []) out_dict = {} for name, out_dims in dims_from_out_properties.items(): - if '*' in out_dims and wildcard_names is not None: - _, out_shape = fill_dims_wildcard( - out_dims, dim_lengths, wildcard_names, expand_wildcard=False) - elif '*' in out_dims and wildcard_names is None: - raise InvalidPropertyDictError( - 'Cannot determine wildcard dimensions required for output if ' - 'there are no wildcard dimensions in input_properties') - else: - out_shape = [] - for dim in out_dims: - out_shape.append(dim_lengths[dim]) + out_shape = [] + for dim in out_dims: + out_shape.append(dim_lengths[dim]) out_dict[name] = np.zeros(out_shape, dtype=dtype) return out_dict +def get_dim_lengths_from_raw_input(raw_input, input_properties): + dim_lengths = {} + for name, properties in input_properties.items(): + for i, dim_name in enumerate(properties['dims']): + if dim_name in dim_lengths: + if raw_input[name].shape[i] != dim_lengths[dim_name]: + raise InvalidStateError( + 'Dimension name {} has differing lengths on different ' + 'inputs'.format(dim_name) + ) + else: + dim_lengths[dim_name] = raw_input[name].shape[i] + return dim_lengths + + def ensure_values_are_arrays(array_dict): for name, value in array_dict.items(): if not isinstance(value, np.ndarray): @@ -331,6 +337,9 @@ def restore_data_arrays_with_properties( continue raw_name = get_alias_or_name(name, output_properties, input_properties) if '*' in out_dims: + for dim_name, length in zip(out_dims, raw_arrays[raw_name].shape): + if dim_name not in dim_lengths and dim_name != '*': + dim_lengths[dim_name] = length out_dims_without_wildcard, target_shape = fill_dims_wildcard( out_dims, dim_lengths, wildcard_names) out_array = expand_array_wildcard_dims( diff --git a/tests/test_get_restore_numpy_array.py b/tests/test_get_restore_numpy_array.py index f46a839..c148b64 100644 --- a/tests/test_get_restore_numpy_array.py +++ b/tests/test_get_restore_numpy_array.py @@ -7,53 +7,6 @@ import numpy as np import unittest -""" -get_numpy_arrays_with_properties: - - returns numpy arrays in the dict - - those numpy arrays should have same dtype as original data - * even when unit conversion happens - - properly collects dimensions along a direction - - they should actually be the same numpy arrays (with memory) as original data if no conversion happens - * even when units are specified (so long as they match) - - should be the same quantities as requested by the properties - * contain all - * not contain extra - * raise exception if some are missing - - units - * converts if requested and present - * does nothing if not requested whether or not present - * raises exception if not requested or not present - * unit conversion should not modify the input array - - requires "dims" to be specified, raises exception if they aren't - - match_dims_like - * should work if matched dimensions are identical - * should raise exception if matched dimensions are not identical - * should require value to be a quantity in property_dictionary - * should require all A matches to look like B and all B matches to look like A - - should raise ValueError when explicitly specified dimension is not present - - should return a scalar array when called on a scalar DataArray - -Test case for when wildcard dimension doesn't match anything - the error message needs to be much more descriptive - e.g. dims=['x', 'y', 'z'] and state=['foo', 'y', 'z'] - -restore_data_arrays_with_properties: - - should return a dictionary of DataArrays - - DataArray values should be the same arrays as original data if no conversion happens - - properly restores collected dimensions - - if conversion does happen, dtype should be the same as the input - - should return same quantities as requested by the properties - * contain all - * not contain extra - * raise exception if some are missing - - units - * should be the same value as specified in output_properties dict - - requires dims_like to be specified, raises exception if it's not - * returned DataArray should have same dimensions as dims_like object - * exception should be raised if dims_like is wrong (shape is incompatible) - * should return coords like the dims_like quantity - -Should add any created exceptions to the docstrings for these functions -""" def test_get_numpy_array_3d_no_change(): array = DataArray( @@ -866,6 +819,64 @@ def test_raises_when_quantity_has_extra_dim_and_unmatched_wildcard(self): else: raise AssertionError('should have raised InvalidStateError') + def test_expands_named_dimension(self): + random = np.random.RandomState(0) + T_array = random.randn(3) + input_state = { + 'air_pressure': DataArray( + np.zeros([3, 4]), + dims=['dim1', 'dim2'], + attrs={'units': 'Pa'}, + ), + 'air_temperature': DataArray( + T_array, + dims=['dim1'], + attrs={'units': 'degK'}, + ) + } + input_properties = { + 'air_pressure': { + 'dims': ['dim1', 'dim2'], + 'units': 'Pa', + }, + 'air_temperature': { + 'dims': ['dim1', 'dim2'], + 'units': 'degK', + }, + } + return_value = get_numpy_arrays_with_properties(input_state, input_properties) + assert return_value['air_temperature'].shape == (3, 4) + assert np.all(return_value['air_temperature'] == T_array[:, None]) + + def test_expands_named_dimension_with_wildcard_present(self): + random = np.random.RandomState(0) + T_array = random.randn(3) + input_state = { + 'air_pressure': DataArray( + np.zeros([3, 4]), + dims=['dim1', 'dim2'], + attrs={'units': 'Pa'}, + ), + 'air_temperature': DataArray( + T_array, + dims=['dim1'], + attrs={'units': 'degK'}, + ) + } + input_properties = { + 'air_pressure': { + 'dims': ['*', 'dim2'], + 'units': 'Pa', + }, + 'air_temperature': { + 'dims': ['*', 'dim2'], + 'units': 'degK', + }, + } + return_value = get_numpy_arrays_with_properties(input_state, input_properties) + assert return_value['air_temperature'].shape == (3, 4) + assert np.all(return_value['air_temperature'] == T_array[:, None]) + class RestoreDataArraysWithPropertiesTests(unittest.TestCase): @@ -1173,6 +1184,64 @@ def test_restores_using_alias_from_input(self): data_arrays['air_pressure'].values) == np.byte_bounds( raw_arrays['p']) + def test_restores_new_dims(self): + input_state = {} + input_properties = {} + raw_arrays = { + 'air_pressure': np.zeros([2, 2, 4]) + } + output_properties = { + 'air_pressure': { + 'dims': ['x', 'y', 'z'], + 'units': 'm', + }, + } + data_arrays = restore_data_arrays_with_properties( + raw_arrays, output_properties, input_state, input_properties + ) + assert len(data_arrays.keys()) == 1 + assert 'air_pressure' in data_arrays.keys() + assert np.all(data_arrays['air_pressure'].values == raw_arrays['air_pressure']) + assert np.byte_bounds( + data_arrays['air_pressure'].values) == np.byte_bounds( + raw_arrays['air_pressure']) + + def test_restores_new_dims_with_wildcard(self): + input_state = { + 'air_pressure': DataArray( + np.zeros([2, 2, 4]), + dims=['x', 'y', 'z'], + attrs={'units': 'degK'}, + ), + } + input_properties = { + 'air_pressure': { + 'dims': ['*'], + 'units': 'degK', + 'alias': 'p' + }, + } + raw_arrays = { + 'q': np.zeros([16, 2]) + } + output_properties = { + 'q': { + 'dims': ['*', 'new_dim'], + 'units': 'm', + }, + } + data_arrays = restore_data_arrays_with_properties( + raw_arrays, output_properties, input_state, input_properties + ) + assert len(data_arrays.keys()) == 1 + assert 'q' in data_arrays.keys() + assert np.all(data_arrays['q'].values.flatten() == raw_arrays['q'].flatten()) + assert np.byte_bounds( + data_arrays['q'].values) == np.byte_bounds( + raw_arrays['q']) + assert data_arrays['q'].dims == ('x', 'y', 'z', 'new_dim') + assert data_arrays['q'].shape == (2, 2, 4, 2) + if __name__ == '__main__': pytest.main([__file__]) From 7d17257b0990616c5ce92f7e297e3ecfee9f463d Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 13 Jun 2018 17:23:39 -0700 Subject: [PATCH 64/98] Fixed array initialization function to use raw state --- tests/test_state.py | 154 ++------------------------------------------ 1 file changed, 7 insertions(+), 147 deletions(-) diff --git a/tests/test_state.py b/tests/test_state.py index 999afc5..7514283 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -30,11 +30,7 @@ def test_single_output_single_dim(self): } } input_state = { - 'input1': DataArray( - np.zeros([10]), - dims=['dim1'], - attrs={'units': 's^-1'} - ) + 'input1': np.zeros([10]) } result = initialize_numpy_arrays_with_properties( @@ -58,11 +54,7 @@ def test_single_output_two_dims(self): } } input_state = { - 'input1': DataArray( - np.zeros([3, 7]), - dims=['dim1', 'dim2'], - attrs={'units': 's^-1'} - ) + 'input1': np.zeros([3, 7]) } result = initialize_numpy_arrays_with_properties( @@ -86,11 +78,7 @@ def test_single_output_two_dims_opposite_order(self): } } input_state = { - 'input1': DataArray( - np.zeros([3, 7]), - dims=['dim1', 'dim2'], - attrs={'units': 's^-1'} - ) + 'input1': np.zeros([3, 7]) } result = initialize_numpy_arrays_with_properties( @@ -118,11 +106,7 @@ def test_two_outputs(self): } } input_state = { - 'input1': DataArray( - np.zeros([3, 7]), - dims=['dim1', 'dim2'], - attrs={'units': 's^-1'} - ) + 'input1': np.zeros([3, 7]) } result = initialize_numpy_arrays_with_properties( @@ -153,16 +137,8 @@ def test_two_inputs(self): } } input_state = { - 'input1': DataArray( - np.zeros([3, 7]), - dims=['dim1', 'dim2'], - attrs={'units': 's^-1'} - ), - 'input2': DataArray( - np.zeros([7, 3]), - dims=['dim2', 'dim1'], - attrs={'units': 's^-1'} - ) + 'input1': np.zeros([3, 7]), + 'input2': np.zeros([7, 3]), } result = initialize_numpy_arrays_with_properties( @@ -186,11 +162,7 @@ def test_single_dim_wildcard(self): } } input_state = { - 'input1': DataArray( - np.zeros([10]), - dims=['dim1'], - attrs={'units': 's^-1'} - ) + 'input1': np.zeros([10]) } result = initialize_numpy_arrays_with_properties( @@ -199,115 +171,3 @@ def test_single_dim_wildcard(self): assert 'output1' in result.keys() assert result['output1'].shape == (10,) assert np.all(result['output1'] == np.zeros([10])) - - def test_two_dims_in_wildcard(self): - output_properties = { - 'output1': { - 'dims': ['*'], - 'units': 'm', - } - } - input_properties = { - 'input1': { - 'dims': ['*'], - 'units': 's^-1', - } - } - input_state = { - 'input1': DataArray( - np.zeros([4, 3]), - dims=['dim1', 'dim2'], - attrs={'units': 's^-1'} - ) - } - - result = initialize_numpy_arrays_with_properties( - output_properties, input_state, input_properties) - assert len(result.keys()) == 1 - assert 'output1' in result.keys() - assert result['output1'].shape == (12,) - assert np.all(result['output1'] == np.zeros([12])) - - def test_two_dims_in_wildcard_with_basic_dim(self): - output_properties = { - 'output1': { - 'dims': ['*', 'dim3'], - 'units': 'm', - } - } - input_properties = { - 'input1': { - 'dims': ['*', 'dim3'], - 'units': 's^-1', - } - } - input_state = { - 'input1': DataArray( - np.zeros([4, 3, 2]), - dims=['dim1', 'dim2', 'dim3'], - attrs={'units': 's^-1'} - ) - } - - result = initialize_numpy_arrays_with_properties( - output_properties, input_state, input_properties) - assert len(result.keys()) == 1 - assert 'output1' in result.keys() - assert result['output1'].shape == (12, 2) - assert np.all(result['output1'] == np.zeros([12, 2])) - - def test_two_dims_in_wildcard_with_basic_dim_in_center(self): - output_properties = { - 'output1': { - 'dims': ['*', 'dim2'], - 'units': 'm', - } - } - input_properties = { - 'input1': { - 'dims': ['*', 'dim2'], - 'units': 's^-1', - } - } - input_state = { - 'input1': DataArray( - np.zeros([4, 3, 2]), - dims=['dim1', 'dim2', 'dim3'], - attrs={'units': 's^-1'} - ) - } - - result = initialize_numpy_arrays_with_properties( - output_properties, input_state, input_properties) - assert len(result.keys()) == 1 - assert 'output1' in result.keys() - assert result['output1'].shape == (8, 3) - assert np.all(result['output1'] == np.zeros([8, 3])) - - def test_two_dims_in_wildcard_with_basic_dim_and_extra_basic_dim_in_input(self): - output_properties = { - 'output1': { - 'dims': ['*', 'dim3'], - 'units': 'm', - } - } - input_properties = { - 'input1': { - 'dims': ['*', 'dim3', 'dim4'], - 'units': 's^-1', - } - } - input_state = { - 'input1': DataArray( - np.zeros([4, 3, 2, 5]), - dims=['dim1', 'dim2', 'dim3', 'dim4'], - attrs={'units': 's^-1'} - ) - } - - result = initialize_numpy_arrays_with_properties( - output_properties, input_state, input_properties) - assert len(result.keys()) == 1 - assert 'output1' in result.keys() - assert result['output1'].shape == (12, 2) - assert np.all(result['output1'] == np.zeros([12, 2])) From e7c60d256f264928c6e31bd9f25dc3b3ff509481 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 13 Jun 2018 17:24:25 -0700 Subject: [PATCH 65/98] flake8 --- sympl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sympl/__init__.py b/sympl/__init__.py index ad19273..65d8cd0 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -47,7 +47,7 @@ restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, initialize_numpy_arrays_with_properties, - get_component_aliases, + get_component_aliases, combine_component_properties, PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, UpdateFrequencyWrapper, ScalingWrapper, From 0400322972a30d031d4e8f9cde422b0348f8c000 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 14 Jun 2018 12:10:15 -0700 Subject: [PATCH 66/98] Added useful error if components have missing or incorrectly typed properties dicts --- sympl/_core/base_components.py | 48 ++++++++++++++++++++++++++++++++++ tests/test_base_components.py | 16 ++++++++++++ 2 files changed, 64 insertions(+) diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index eff7993..823953a 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -109,6 +109,18 @@ class InputChecker(object): def __init__(self, component): self.component = component + if not hasattr(component, 'input_properties'): + raise InvalidPropertyDictError( + 'Component of type {} is missing input_properties'.format( + component.__class__.__name__) + ) + elif not isinstance(component.input_properties, dict): + raise InvalidPropertyDictError( + 'input_properties on component of type {} is of type {}, but ' + 'should be an instance of dict'.format( + component.__class__.__name__, + component.input_properties.__class__) + ) for name, properties in self.component.input_properties.items(): if 'units' not in properties.keys(): raise InvalidPropertyDictError( @@ -160,6 +172,18 @@ class TendencyChecker(object): def __init__(self, component): self.component = component + if not hasattr(component, 'tendency_properties'): + raise InvalidPropertyDictError( + 'Component of type {} is missing tendency_properties'.format( + component.__class__.__name__) + ) + elif not isinstance(component.tendency_properties, dict): + raise InvalidPropertyDictError( + 'tendency_properties on component of type {} is of type {}, but ' + 'should be an instance of dict'.format( + component.__class__.__name__, + component.input_properties.__class__) + ) for name, properties in self.component.tendency_properties.items(): if 'units' not in properties.keys(): raise InvalidPropertyDictError( @@ -224,6 +248,18 @@ class DiagnosticChecker(object): def __init__(self, component): self.component = component + if not hasattr(component, 'diagnostic_properties'): + raise InvalidPropertyDictError( + 'Component of type {} is missing diagnostic_properties'.format( + component.__class__.__name__) + ) + elif not isinstance(component.diagnostic_properties, dict): + raise InvalidPropertyDictError( + 'diagnostic_properties on component of type {} is of type {}, but ' + 'should be an instance of dict'.format( + component.__class__.__name__, + component.input_properties.__class__) + ) self._ignored_diagnostics = [] for name, properties in component.diagnostic_properties.items(): if 'units' not in properties.keys(): @@ -293,6 +329,18 @@ class OutputChecker(object): def __init__(self, component): self.component = component + if not hasattr(component, 'output_properties'): + raise InvalidPropertyDictError( + 'Component of type {} is missing output_properties'.format( + component.__class__.__name__) + ) + elif not isinstance(component.output_properties, dict): + raise InvalidPropertyDictError( + 'output_properties on component of type {} is of type {}, but ' + 'should be an instance of dict'.format( + component.__class__.__name__, + component.input_properties.__class__) + ) for name, properties in self.component.output_properties.items(): if 'units' not in properties.keys(): raise InvalidPropertyDictError( diff --git a/tests/test_base_components.py b/tests/test_base_components.py index 7c37b8f..3f2e729 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -170,6 +170,10 @@ def array_call(self, state, timestep): class InputTestBase(): + def test_raises_on_input_properties_of_wrong_type(self): + with self.assertRaises(InvalidPropertyDictError): + self.get_component(input_properties=({},)) + def test_cannot_overlap_input_aliases(self): input_properties = { 'input1': {'dims': ['dim1'], 'units': 'm', 'alias': 'input'}, @@ -438,6 +442,10 @@ def test_input_is_aliased(self): class DiagnosticTestBase(): + def test_raises_on_diagnostic_properties_of_wrong_type(self): + with self.assertRaises(InvalidPropertyDictError): + self.get_component(diagnostic_properties=({},)) + def test_diagnostic_requires_dims(self): diagnostic_properties = {'diag1': {'units': 'm'}} with self.assertRaises(InvalidPropertyDictError): @@ -745,6 +753,10 @@ def get_component( def get_diagnostics(self, result): return result[1] + def test_raises_on_tendency_properties_of_wrong_type(self): + with self.assertRaises(InvalidPropertyDictError): + self.get_component(tendency_properties=({},)) + def test_cannot_use_bad_component(self): component = BadMockPrognostic() with self.assertRaises(RuntimeError): @@ -1511,6 +1523,10 @@ def get_component( def get_diagnostics(self, result): return result[0] + def test_raises_on_output_properties_of_wrong_type(self): + with self.assertRaises(InvalidPropertyDictError): + self.get_component(output_properties=({},)) + def test_cannot_use_bad_component(self): component = BadMockImplicit() with self.assertRaises(RuntimeError): From f4fc9e86ec2e6ce089382dbd3593cfa685bdcc2a Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 14 Jun 2018 12:48:23 -0700 Subject: [PATCH 67/98] Allow dtype in properties to determine array dtype initialized by helper function --- sympl/_core/state.py | 3 ++- tests/test_state.py | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/sympl/_core/state.py b/sympl/_core/state.py index d8486a2..6d6b168 100644 --- a/sympl/_core/state.py +++ b/sympl/_core/state.py @@ -146,7 +146,7 @@ def get_numpy_array(data_array, out_dims, dim_lengths): def initialize_numpy_arrays_with_properties( - output_properties, raw_input_state, input_properties, dtype=np.float64): + output_properties, raw_input_state, input_properties): """ Parameters ---------- @@ -185,6 +185,7 @@ def initialize_numpy_arrays_with_properties( out_shape = [] for dim in out_dims: out_shape.append(dim_lengths[dim]) + dtype = output_properties[name].get('dtype', np.float64) out_dict[name] = np.zeros(out_shape, dtype=dtype) return out_dict diff --git a/tests/test_state.py b/tests/test_state.py index 7514283..aaa6040 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -39,6 +39,33 @@ def test_single_output_single_dim(self): assert 'output1' in result.keys() assert result['output1'].shape == (10,) assert np.all(result['output1'] == np.zeros([10])) + assert result['output1'].dtype == np.float64 + + def test_single_output_single_dim_custom_dtype(self): + output_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + 'dtype': np.int32, + } + } + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 's^-1', + } + } + input_state = { + 'input1': np.zeros([10]) + } + + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert len(result.keys()) == 1 + assert 'output1' in result.keys() + assert result['output1'].shape == (10,) + assert np.all(result['output1'] == np.zeros([10])) + assert result['output1'].dtype == np.int32 def test_single_output_two_dims(self): output_properties = { From b695e76a79df11446f23822303f4c97973671974 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 15 Jun 2018 14:20:46 -0700 Subject: [PATCH 68/98] Handle ImplicitPrognostic components in sympl TimeSteppers --- HISTORY.rst | 1 + sympl/_components/timesteppers.py | 10 +- sympl/_core/composite.py | 76 ++- sympl/_core/timestepper.py | 22 +- tests/test_timestepping.py | 1058 ++++++++++++++++------------- 5 files changed, 673 insertions(+), 494 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b322613..f236c33 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,7 @@ Latest composited. * TimeSteppers now have a prognostic_list attribute which contains the prognostics used to calculate tendencies. +* TimeSteppers from sympl can now handle ImplicitPrognostic components. * Added a check for netcdftime having the required objects, to fall back on not using netcdftime when those are missing. This is because most objects are missing in older versions of netcdftime (that come packaged with netCDF4) (closes #23). diff --git a/sympl/_components/timesteppers.py b/sympl/_components/timesteppers.py index 2808f8f..098e599 100644 --- a/sympl/_components/timesteppers.py +++ b/sympl/_components/timesteppers.py @@ -16,7 +16,7 @@ def __init__(self, *args, **kwargs): Args ---- - *args : Prognostic + *args : Prognostic or ImplicitPrognostic Objects to call for tendencies when doing time stepping. stages: int, optional Number of stages to use. Should be 2 or 3. Default is 3. @@ -80,7 +80,7 @@ def __init__(self, *args, **kwargs): Args ---- - *args : Prognostic + *args : Prognostic or ImplicitPrognostic Objects to call for tendencies when doing time stepping. order : int, optional The order of accuracy to use. Must be between @@ -126,7 +126,7 @@ def _call(self, state, timestep): """ self._ensure_constant_timestep(timestep) state = state.copy() - tendencies, diagnostics = self.prognostic(state) + tendencies, diagnostics = self.prognostic(state, timestep) convert_tendencies_units_for_state(tendencies, state) self._tendencies_list.append(tendencies) new_state = self._perform_step(state, timestep) @@ -190,7 +190,7 @@ def __init__(self, *args, **kwargs): Args ---- - *args : Prognostic + *args : Prognostic or ImplicitPrognostic Objects to call for tendencies when doing time stepping. asselin_strength : float, optional The filter parameter used to determine the strength @@ -247,7 +247,7 @@ def _call(self, state, timestep): original_state = state state = state.copy() self._ensure_constant_timestep(timestep) - tendencies, diagnostics = self.prognostic(state) + tendencies, diagnostics = self.prognostic(state, timestep) convert_tendencies_units_for_state(tendencies, state) if self._old_state is None: new_state = step_forward_euler(state, tendencies, timestep) diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 925baf0..9ad6a3d 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -1,4 +1,4 @@ -from .base_components import Prognostic, Diagnostic, Monitor +from .base_components import Prognostic, Diagnostic, Monitor, ImplicitPrognostic from .util import ( update_dict_by_adding_another, ensure_no_shared_keys, combine_component_properties) @@ -170,6 +170,80 @@ def array_call(self, state): raise NotImplementedError() +class ImplicitPrognosticComposite(ComponentComposite, InputPropertiesCompositeMixin, + DiagnosticPropertiesCompositeMixin, ImplicitPrognostic): + + component_class = (Prognostic, ImplicitPrognostic) + + @property + def tendency_properties(self): + return combine_component_properties( + self.component_list, 'tendency_properties', self.input_properties) + + def __init__(self, *args): + """ + Args + ---- + *args + The components that should be wrapped by this object. + + Raises + ------ + SharedKeyError + If two components compute the same diagnostic quantity. + InvalidPropertyDictError + If two components require the same input or compute the same + output quantity, and their dimensions or units are incompatible + with one another. + """ + super(ImplicitPrognosticComposite, self).__init__(*args) + self.input_properties + self.tendency_properties + self.diagnostic_properties + + def __call__(self, state, timestep): + """ + Gets tendencies and diagnostics from the passed model state. + + Args + ---- + state : dict + A model state dictionary. + + Returns + ------- + tendencies : dict + A dictionary whose keys are strings indicating + state quantities and values are the time derivative of those + quantities in units/second at the time of the input state. + diagnostics : dict + A dictionary whose keys are strings indicating + state quantities and values are the value of those quantities + at the time of the input state. + + Raises + ------ + KeyError + If a required quantity is missing from the state. + InvalidStateError + If state is not a valid input for a Prognostic instance. + """ + return_tendencies = {} + return_diagnostics = {} + for prognostic in self.component_list: + if isinstance(prognostic, ImplicitPrognostic): + tendencies, diagnostics = prognostic(state, timestep) + elif isinstance(prognostic, Prognostic): + tendencies, diagnostics = prognostic(state) + update_dict_by_adding_another(return_tendencies, tendencies) + return_diagnostics.update(diagnostics) + return return_tendencies, return_diagnostics + + def array_call(self, state): + raise NotImplementedError() + + + class DiagnosticComposite( ComponentComposite, InputPropertiesCompositeMixin, DiagnosticPropertiesCompositeMixin, Diagnostic): diff --git a/sympl/_core/timestepper.py b/sympl/_core/timestepper.py index ac8ddbc..9048010 100644 --- a/sympl/_core/timestepper.py +++ b/sympl/_core/timestepper.py @@ -1,9 +1,10 @@ import abc -from .composite import PrognosticComposite +from .composite import ImplicitPrognosticComposite from .time import timedelta from .util import combine_component_properties, combine_properties from .units import clean_units from .state import copy_untouched_quantities +from .base_components import ImplicitPrognostic import warnings @@ -25,9 +26,10 @@ class TimeStepper(object): for the new state are returned when the object is called, and values are dictionaries which indicate 'dims' and 'units'. - prognostic : PrognosticComposite - A composite of the Prognostic objects used by the TimeStepper - prognostic_list: list of Prognostic + prognostic : ImplicitPrognosticComposite + A composite of the Prognostic and ImplicitPrognostic objects used by + the TimeStepper. + prognostic_list: list of Prognostic and ImplicitPrognostic A list of Prognostic objects called by the TimeStepper. These should be referenced when determining what inputs are necessary for the TimeStepper. @@ -114,7 +116,7 @@ def __init__(self, *args, **kwargs): Parameters ---------- - *args : Prognostic + *args : Prognostic or ImplicitPrognostic Objects to call for tendencies when doing time stepping. tendencies_in_diagnostics : bool, optional A boolean indicating whether this object will put tendencies of @@ -133,10 +135,16 @@ def __init__(self, *args, **kwargs): if len(args) == 1 and isinstance(args[0], list): warnings.warn( 'TimeSteppers should be given individual Prognostics rather ' - 'than a list, and will not accept lists in a later version.') + 'than a list, and will not accept lists in a later version.', + DeprecationWarning) args = args[0] self._tendencies_in_diagnostics = tendencies_in_diagnostics - self.prognostic = PrognosticComposite(*args) + # warnings.simplefilter('always') + if any(isinstance(a, ImplicitPrognostic) for a in args): + print('WARNING now') + warnings.warn('ImplicitPrognostic') + print(warnings) + self.prognostic = ImplicitPrognosticComposite(*args) @property def prognostic_list(self): diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index ecfacb7..44ead14 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -2,9 +2,12 @@ import mock from sympl import ( Prognostic, Leapfrog, AdamsBashforth, DataArray, SSPRungeKutta, timedelta, - InvalidPropertyDictError) + InvalidPropertyDictError, ImplicitPrognostic) from sympl._core.units import units_are_compatible import numpy as np +import warnings + +warnings.filterwarnings('always', 'ImplicitPrognostic') def same_list(list1, list2): @@ -46,137 +49,508 @@ def array_call(self, state): return self._tendency_output, self._diagnostic_output -class TimesteppingBase(object): - - timestepper_class = None +class MockImplicitPrognostic(ImplicitPrognostic): + input_properties = None + diagnostic_properties = None + tendency_properties = None - def test_unused_quantities_carried_over(self): - state = {'time': timedelta(0), 'air_temperature': 273.} - time_stepper = self.timestepper_class(MockEmptyPrognostic()) - timestep = timedelta(seconds=1.) - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'time': timedelta(0), 'air_temperature': 273.} - assert new_state == {'time': timedelta(0), 'air_temperature': 273.} + def __init__( + self, input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output, **kwargs): + self.input_properties = input_properties + self.diagnostic_properties = diagnostic_properties + self.tendency_properties = tendency_properties + self._diagnostic_output = diagnostic_output + self._tendency_output = tendency_output + self.times_called = 0 + self.state_given = None + super(MockImplicitPrognostic, self).__init__(**kwargs) - def test_timestepper_reveals_prognostics(self): - prog1 = MockEmptyPrognostic() - prog1.input_properties = {'input1': {'dims': ['dim1'], 'units': 'm'}} - time_stepper = self.timestepper_class(prog1) - assert same_list(time_stepper.prognostic_list, (prog1,)) + def array_call(self, state, timestep): + self.times_called += 1 + self.state_given = state + return self._tendency_output, self._diagnostic_output - @mock.patch.object(MockEmptyPrognostic, '__call__') - def test_float_no_change_one_step(self, mock_prognostic_call): - mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) - state = {'time': timedelta(0), 'air_temperature': 273.} - timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'time': timedelta(0), 'air_temperature': 273.} - assert new_state == {'time': timedelta(0), 'air_temperature': 273.} - assert diagnostics == {} - @mock.patch.object(MockEmptyPrognostic, '__call__') - def test_float_no_change_one_step_diagnostic(self, mock_prognostic_call): - mock_prognostic_call.return_value = ( - {'air_temperature': 0.}, {'foo': 'bar'}) - state = {'time': timedelta(0), 'air_temperature': 273.} - timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'time': timedelta(0), 'air_temperature': 273.} - assert new_state == {'time': timedelta(0), 'air_temperature': 273.} - assert diagnostics == {'foo': 'bar'} +class PrognosticBase(object): - @mock.patch.object(MockEmptyPrognostic, '__call__') - def test_float_no_change_three_steps(self, mock_prognostic_call): - mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) - state = {'time': timedelta(0), 'air_temperature': 273.} - timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'time': timedelta(0), 'air_temperature': 273.} - assert new_state == {'time': timedelta(0), 'air_temperature': 273.} - state = new_state - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'time': timedelta(0), 'air_temperature': 273.} - assert new_state == {'time': timedelta(0), 'air_temperature': 273.} - state = new_state - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'time': timedelta(0), 'air_temperature': 273.} - assert new_state == {'time': timedelta(0), 'air_temperature': 273.} + prognostic_class = MockPrognostic - @mock.patch.object(MockEmptyPrognostic, '__call__') - def test_float_one_step(self, mock_prognostic_call): - mock_prognostic_call.return_value = ({'air_temperature': 1.}, {}) - state = {'time': timedelta(0), 'air_temperature': 273.} - timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'time': timedelta(0), 'air_temperature': 273.} - assert new_state == {'time': timedelta(0), 'air_temperature': 274.} + def test_given_tendency_not_modified_with_two_components(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'tend1': { + 'dims': ['dim1'], + 'units': 'm/s', + } + } + diagnostic_output = {} + tendency_output_1 = { + 'tend1': np.ones([10]) * 5. + } + prognostic1 = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output_1 + ) + tendency_output_2 = { + 'tend1': np.ones([10]) * 5. + } + prognostic2 = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output_2 + ) + stepper = self.timestepper_class( + prognostic1, prognostic2, tendencies_in_diagnostics=True) + state = { + 'time': timedelta(0), + 'tend1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'}, + ) + } + _, _ = stepper(state, timedelta(seconds=5)) + assert np.all(tendency_output_1['tend1'] == 5.) + assert np.all(tendency_output_2['tend1'] == 5.) - @mock.patch.object(MockEmptyPrognostic, '__call__') - def test_float_one_step_with_units(self, mock_prognostic_call): - mock_prognostic_call.return_value = ({'eastward_wind': DataArray(0.02, attrs={'units': 'km/s^2'})}, {}) - state = {'time': timedelta(0), 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} - timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'time': timedelta(0), 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} - assert same_list(new_state.keys(), ['time', 'eastward_wind']) - assert np.allclose(new_state['eastward_wind'].values, 21.) - assert new_state['eastward_wind'].attrs['units'] == 'm/s' + def test_input_state_not_modified_with_two_components(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'tend1': { + 'dims': ['dim1'], + 'units': 'm/s', + } + } + diagnostic_output = {} + tendency_output_1 = { + 'tend1': np.ones([10]) * 5. + } + prognostic1 = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output_1 + ) + tendency_output_2 = { + 'tend1': np.ones([10]) * 5. + } + prognostic2 = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output_2 + ) + stepper = self.timestepper_class( + prognostic1, prognostic2, tendencies_in_diagnostics=True) + state = { + 'time': timedelta(0), + 'tend1': DataArray( + np.ones([10]), + dims=['dim1'], + attrs={'units': 'm'}, + ) + } + _, _ = stepper(state, timedelta(seconds=5)) + assert state['tend1'].attrs['units'] == 'm' + assert np.all(state['tend1'].values == 1.) - @mock.patch.object(MockEmptyPrognostic, '__call__') - def test_float_three_steps(self, mock_prognostic_call): - mock_prognostic_call.return_value = ({'air_temperature': 1.}, {}) - state = {'time': timedelta(0), 'air_temperature': 273.} - timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'time': timedelta(0), 'air_temperature': 273.} - assert new_state == {'time': timedelta(0), 'air_temperature': 274.} - state = new_state - diagnostics, new_state = time_stepper.__call__(new_state, timestep) - assert state == {'time': timedelta(0), 'air_temperature': 274.} - assert new_state == {'time': timedelta(0), 'air_temperature': 275.} - state = new_state - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert state == {'time': timedelta(0), 'air_temperature': 275.} - assert new_state == {'time': timedelta(0), 'air_temperature': 276.} + def test_tendencies_in_diagnostics_no_tendency(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = {} + diagnostic_output = {} + tendency_output = {} + prognostic = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class( + prognostic, tendencies_in_diagnostics=True) + state = {'time': timedelta(0)} + diagnostics, _ = stepper(state, timedelta(seconds=5)) + assert diagnostics == {} - @mock.patch.object(MockEmptyPrognostic, '__call__') - def test_array_no_change_one_step(self, mock_prognostic_call): - mock_prognostic_call.return_value = ( - {'air_temperature': np.zeros((3, 3))}, {}) - state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} - timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert same_list(state.keys(), ['time', 'air_temperature']) - assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert same_list(new_state.keys(), ['time', 'air_temperature']) - assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() + def test_tendencies_in_diagnostics_one_tendency(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + } + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class( + prognostic, tendencies_in_diagnostics=True) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10])*10., + dims=['dim1'], + attrs={'units': 'm'} + ), + } + diagnostics, _ = stepper(state, timedelta(seconds=5)) + tendency_name = 'output1_tendency_from_{}'.format(stepper.__class__.__name__) + assert tendency_name in diagnostics.keys() + assert len( + diagnostics[tendency_name].dims) == 1 + assert 'dim1' in diagnostics[tendency_name].dims + assert units_are_compatible(diagnostics[tendency_name].attrs['units'], 'm s^-1') + assert np.allclose(diagnostics[tendency_name].values, 2.) - @mock.patch.object(MockEmptyPrognostic, '__call__') - def test_array_no_change_three_steps(self, mock_prognostic_call): - mock_prognostic_call.return_value = ( - {'air_temperature': np.ones((3, 3))*0.}, {}) - state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} - timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert same_list(state.keys(), ['time', 'air_temperature']) - assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert same_list(new_state.keys(), ['time', 'air_temperature']) - assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() - state = new_state - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert same_list(state.keys(), ['time', 'air_temperature']) - assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert same_list(new_state.keys(), ['time', 'air_temperature']) - assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() - state = new_state + def test_tendencies_in_diagnostics_one_tendency_with_component_name(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + } + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class( + prognostic, tendencies_in_diagnostics=True, name='component') + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10])*10., + dims=['dim1'], + attrs={'units': 'm'} + ), + } + diagnostics, _ = stepper(state, timedelta(seconds=5)) + assert 'output1_tendency_from_component' in diagnostics.keys() + assert len( + diagnostics['output1_tendency_from_component'].dims) == 1 + assert 'dim1' in diagnostics['output1_tendency_from_component'].dims + assert units_are_compatible(diagnostics['output1_tendency_from_component'].attrs['units'], 'm s^-1') + assert np.allclose(diagnostics['output1_tendency_from_component'].values, 2.) + + def test_copies_untouched_quantities(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + }, + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class( + prognostic, tendencies_in_diagnostics=True, name='component') + untouched_quantity = DataArray( + np.ones([10])*10., + dims=['dim1'], + attrs={'units': 'J'} + ) + state = { + 'time': timedelta(0), + 'output1': DataArray( + np.ones([10])*10., + dims=['dim1'], + attrs={'units': 'm'} + ), + 'input1': untouched_quantity, + } + _, new_state = stepper(state, timedelta(seconds=5)) + assert 'input1' in new_state.keys() + assert new_state['input1'].dims == untouched_quantity.dims + assert np.allclose(new_state['input1'].values, 10.) + assert new_state['input1'].attrs['units'] == 'J' + + def test_stepper_requires_input_for_stepped_quantity(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + }, + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class(prognostic) + assert 'output1' in stepper.input_properties.keys() + assert stepper.input_properties['output1']['dims'] == ['dim1'] + assert units_are_compatible(stepper.input_properties['output1']['units'], 'm') + + def test_stepper_outputs_stepped_quantity(self): + input_properties = {} + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + }, + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class(prognostic) + assert 'output1' in stepper.output_properties.keys() + assert stepper.output_properties['output1']['dims'] == ['dim1'] + assert units_are_compatible(stepper.output_properties['output1']['units'], 'm') + + def test_stepper_requires_input_for_input_quantity(self): + input_properties = { + 'input1': { + 'dims': ['dim1', 'dim2'], + 'units': 's', + } + } + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + }, + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class(prognostic) + assert 'input1' in stepper.input_properties.keys() + assert stepper.input_properties['input1']['dims'] == ['dim1', 'dim2'] + assert units_are_compatible(stepper.input_properties['input1']['units'], 's') + assert len(stepper.diagnostic_properties) == 0 + + def test_stepper_gives_diagnostic_quantity(self): + input_properties = {} + diagnostic_properties = { + 'diag1': { + 'dims': ['dim2'], + 'units': '', + } + } + tendency_properties = { + } + diagnostic_output = {} + tendency_output = { + } + prognostic = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class( + prognostic, tendencies_in_diagnostics=True, name='component') + assert 'diag1' in stepper.diagnostic_properties.keys() + assert stepper.diagnostic_properties['diag1']['dims'] == ['dim2'] + assert units_are_compatible( + stepper.diagnostic_properties['diag1']['units'], '') + assert len(stepper.input_properties) == 0 + assert len(stepper.output_properties) == 0 + + def test_stepper_gives_diagnostic_tendency_quantity(self): + input_properties = { + 'input1': { + 'dims': ['dim1', 'dim2'], + 'units': 's', + } + } + diagnostic_properties = {} + tendency_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm/s' + }, + } + diagnostic_output = {} + tendency_output = { + 'output1': np.ones([10]) * 2., + } + prognostic = self.prognostic_class( + input_properties, diagnostic_properties, tendency_properties, + diagnostic_output, tendency_output + ) + stepper = self.timestepper_class( + prognostic, tendencies_in_diagnostics=True) + tendency_name = 'output1_tendency_from_{}'.format(stepper.__class__.__name__) + assert tendency_name in stepper.diagnostic_properties.keys() + assert len(stepper.diagnostic_properties) == 1 + assert stepper.diagnostic_properties[tendency_name]['dims'] == ['dim1'] + assert units_are_compatible(stepper.input_properties['output1']['units'], 'm') + assert units_are_compatible(stepper.diagnostic_properties[tendency_name]['units'], 'm/s') + + +class ImplicitPrognosticBase(PrognosticBase): + + prognostic_class = MockImplicitPrognostic + + def test_warn_on_implicitprognostic(self): + prognostic = self.prognostic_class({}, {}, {}, {}, {}) + with pytest.warns(UserWarning) as w: + self.timestepper_class(prognostic) + print(list(i.message for i in w)) + assert 'ImplicitPrognostic' in w[0].message + + +class TimesteppingBase(object): + + timestepper_class = None + + def test_unused_quantities_carried_over(self): + state = {'time': timedelta(0), 'air_temperature': 273.} + time_stepper = self.timestepper_class(MockEmptyPrognostic()) + timestep = timedelta(seconds=1.) + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} + + def test_timestepper_reveals_prognostics(self): + prog1 = MockEmptyPrognostic() + prog1.input_properties = {'input1': {'dims': ['dim1'], 'units': 'm'}} + time_stepper = self.timestepper_class(prog1) + assert same_list(time_stepper.prognostic_list, (prog1,)) + + @mock.patch.object(MockEmptyPrognostic, '__call__') + def test_float_no_change_one_step(self, mock_prognostic_call): + mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) + state = {'time': timedelta(0), 'air_temperature': 273.} + timestep = timedelta(seconds=1.) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} + assert diagnostics == {} + + @mock.patch.object(MockEmptyPrognostic, '__call__') + def test_float_no_change_one_step_diagnostic(self, mock_prognostic_call): + mock_prognostic_call.return_value = ( + {'air_temperature': 0.}, {'foo': 'bar'}) + state = {'time': timedelta(0), 'air_temperature': 273.} + timestep = timedelta(seconds=1.) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} + assert diagnostics == {'foo': 'bar'} + + @mock.patch.object(MockEmptyPrognostic, '__call__') + def test_float_no_change_three_steps(self, mock_prognostic_call): + mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) + state = {'time': timedelta(0), 'air_temperature': 273.} + timestep = timedelta(seconds=1.) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} + state = new_state + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} + state = new_state + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 273.} + + @mock.patch.object(MockEmptyPrognostic, '__call__') + def test_float_one_step(self, mock_prognostic_call): + mock_prognostic_call.return_value = ({'air_temperature': 1.}, {}) + state = {'time': timedelta(0), 'air_temperature': 273.} + timestep = timedelta(seconds=1.) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 274.} + + @mock.patch.object(MockEmptyPrognostic, '__call__') + def test_float_one_step_with_units(self, mock_prognostic_call): + mock_prognostic_call.return_value = ({'eastward_wind': DataArray(0.02, attrs={'units': 'km/s^2'})}, {}) + state = {'time': timedelta(0), 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} + timestep = timedelta(seconds=1.) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert state == {'time': timedelta(0), 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} + assert same_list(new_state.keys(), ['time', 'eastward_wind']) + assert np.allclose(new_state['eastward_wind'].values, 21.) + assert new_state['eastward_wind'].attrs['units'] == 'm/s' + + @mock.patch.object(MockEmptyPrognostic, '__call__') + def test_float_three_steps(self, mock_prognostic_call): + mock_prognostic_call.return_value = ({'air_temperature': 1.}, {}) + state = {'time': timedelta(0), 'air_temperature': 273.} + timestep = timedelta(seconds=1.) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert state == {'time': timedelta(0), 'air_temperature': 273.} + assert new_state == {'time': timedelta(0), 'air_temperature': 274.} + state = new_state + diagnostics, new_state = time_stepper.__call__(new_state, timestep) + assert state == {'time': timedelta(0), 'air_temperature': 274.} + assert new_state == {'time': timedelta(0), 'air_temperature': 275.} + state = new_state + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert state == {'time': timedelta(0), 'air_temperature': 275.} + assert new_state == {'time': timedelta(0), 'air_temperature': 276.} + + @mock.patch.object(MockEmptyPrognostic, '__call__') + def test_array_no_change_one_step(self, mock_prognostic_call): + mock_prognostic_call.return_value = ( + {'air_temperature': np.zeros((3, 3))}, {}) + state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} + timestep = timedelta(seconds=1.) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert same_list(state.keys(), ['time', 'air_temperature']) + assert (state['air_temperature'] == np.ones((3, 3))*273.).all() + assert same_list(new_state.keys(), ['time', 'air_temperature']) + assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() + + @mock.patch.object(MockEmptyPrognostic, '__call__') + def test_array_no_change_three_steps(self, mock_prognostic_call): + mock_prognostic_call.return_value = ( + {'air_temperature': np.ones((3, 3))*0.}, {}) + state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} + timestep = timedelta(seconds=1.) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert same_list(state.keys(), ['time', 'air_temperature']) + assert (state['air_temperature'] == np.ones((3, 3))*273.).all() + assert same_list(new_state.keys(), ['time', 'air_temperature']) + assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() + state = new_state + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert same_list(state.keys(), ['time', 'air_temperature']) + assert (state['air_temperature'] == np.ones((3, 3))*273.).all() + assert same_list(new_state.keys(), ['time', 'air_temperature']) + assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() + state = new_state diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -333,407 +707,129 @@ def test_dataarray_three_steps(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' - def test_given_tendency_not_modified_with_two_components(self): - input_properties = {} - diagnostic_properties = {} - tendency_properties = { - 'tend1': { - 'dims': ['dim1'], - 'units': 'm/s', - } - } - diagnostic_output = {} - tendency_output_1 = { - 'tend1': np.ones([10]) * 5. - } - prognostic1 = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output_1 - ) - tendency_output_2 = { - 'tend1': np.ones([10]) * 5. - } - prognostic2 = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output_2 - ) - stepper = self.timestepper_class( - prognostic1, prognostic2, tendencies_in_diagnostics=True) - state = { - 'time': timedelta(0), - 'tend1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'}, - ) - } - _, _ = stepper(state, timedelta(seconds=5)) - assert np.all(tendency_output_1['tend1'] == 5.) - assert np.all(tendency_output_2['tend1'] == 5.) - - def test_input_state_not_modified_with_two_components(self): - input_properties = {} - diagnostic_properties = {} - tendency_properties = { - 'tend1': { - 'dims': ['dim1'], - 'units': 'm/s', - } - } - diagnostic_output = {} - tendency_output_1 = { - 'tend1': np.ones([10]) * 5. - } - prognostic1 = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output_1 - ) - tendency_output_2 = { - 'tend1': np.ones([10]) * 5. - } - prognostic2 = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output_2 - ) - stepper = self.timestepper_class( - prognostic1, prognostic2, tendencies_in_diagnostics=True) - state = { - 'time': timedelta(0), - 'tend1': DataArray( - np.ones([10]), - dims=['dim1'], - attrs={'units': 'm'}, - ) - } - _, _ = stepper(state, timedelta(seconds=5)) - assert state['tend1'].attrs['units'] == 'm' - assert np.all(state['tend1'].values == 1.) - - def test_tendencies_in_diagnostics_no_tendency(self): - input_properties = {} - diagnostic_properties = {} - tendency_properties = {} - diagnostic_output = {} - tendency_output = {} - prognostic = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - stepper = self.timestepper_class( - prognostic, tendencies_in_diagnostics=True) - state = {'time': timedelta(0)} - diagnostics, _ = stepper(state, timedelta(seconds=5)) - assert diagnostics == {} - - def test_tendencies_in_diagnostics_one_tendency(self): - input_properties = {} - diagnostic_properties = {} - tendency_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm/s' - } - } - diagnostic_output = {} - tendency_output = { - 'output1': np.ones([10]) * 2., - } - prognostic = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - stepper = self.timestepper_class( - prognostic, tendencies_in_diagnostics=True) - state = { - 'time': timedelta(0), - 'output1': DataArray( - np.ones([10])*10., - dims=['dim1'], - attrs={'units': 'm'} - ), - } - diagnostics, _ = stepper(state, timedelta(seconds=5)) - tendency_name = 'output1_tendency_from_{}'.format(stepper.__class__.__name__) - assert tendency_name in diagnostics.keys() - assert len( - diagnostics[tendency_name].dims) == 1 - assert 'dim1' in diagnostics[tendency_name].dims - assert units_are_compatible(diagnostics[tendency_name].attrs['units'], 'm s^-1') - assert np.allclose(diagnostics[tendency_name].values, 2.) + @mock.patch.object(MockEmptyPrognostic, '__call__') + def test_array_four_steps(self, mock_prognostic_call): + mock_prognostic_call.return_value = ( + {'air_temperature': np.ones((3, 3)) * 1.}, {}) + state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3)) * 273.} + timestep = timedelta(seconds=1.) + time_stepper = self.timestepper_class(MockEmptyPrognostic()) + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert same_list(state.keys(), ['time', 'air_temperature']) + assert (state['air_temperature'] == np.ones((3, 3)) * 273.).all() + assert same_list(new_state.keys(), ['time', 'air_temperature']) + assert (new_state['air_temperature'] == np.ones((3, 3)) * 274.).all() + state = new_state + diagnostics, new_state = time_stepper.__call__(new_state, timestep) + assert same_list(state.keys(), ['time', 'air_temperature']) + assert (state['air_temperature'] == np.ones((3, 3)) * 274.).all() + assert same_list(new_state.keys(), ['time', 'air_temperature']) + assert (new_state['air_temperature'] == np.ones((3, 3)) * 275.).all() + state = new_state + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert same_list(state.keys(), ['time', 'air_temperature']) + assert (state['air_temperature'] == np.ones((3, 3)) * 275.).all() + assert same_list(new_state.keys(), ['time', 'air_temperature']) + assert (new_state['air_temperature'] == np.ones((3, 3)) * 276.).all() + state = new_state + diagnostics, new_state = time_stepper.__call__(state, timestep) + assert same_list(state.keys(), ['time', 'air_temperature']) + assert (state['air_temperature'] == np.ones((3, 3)) * 276.).all() + assert same_list(new_state.keys(), ['time', 'air_temperature']) + assert (new_state['air_temperature'] == np.ones((3, 3)) * 277.).all() - def test_tendencies_in_diagnostics_one_tendency_with_component_name(self): - input_properties = {} - diagnostic_properties = {} - tendency_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm/s' - } - } - diagnostic_output = {} - tendency_output = { - 'output1': np.ones([10]) * 2., - } - prognostic = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - stepper = self.timestepper_class( - prognostic, tendencies_in_diagnostics=True, name='component') - state = { - 'time': timedelta(0), - 'output1': DataArray( - np.ones([10])*10., - dims=['dim1'], - attrs={'units': 'm'} - ), - } - diagnostics, _ = stepper(state, timedelta(seconds=5)) - assert 'output1_tendency_from_component' in diagnostics.keys() - assert len( - diagnostics['output1_tendency_from_component'].dims) == 1 - assert 'dim1' in diagnostics['output1_tendency_from_component'].dims - assert units_are_compatible(diagnostics['output1_tendency_from_component'].attrs['units'], 'm s^-1') - assert np.allclose(diagnostics['output1_tendency_from_component'].values, 2.) - def test_copies_untouched_quantities(self): - input_properties = {} - diagnostic_properties = {} - tendency_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm/s' - }, - } - diagnostic_output = {} - tendency_output = { - 'output1': np.ones([10]) * 2., - } - prognostic = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - stepper = self.timestepper_class( - prognostic, tendencies_in_diagnostics=True, name='component') - untouched_quantity = DataArray( - np.ones([10])*10., - dims=['dim1'], - attrs={'units': 'J'} - ) - state = { - 'time': timedelta(0), - 'output1': DataArray( - np.ones([10])*10., - dims=['dim1'], - attrs={'units': 'm'} - ), - 'input1': untouched_quantity, - } - _, new_state = stepper(state, timedelta(seconds=5)) - assert 'input1' in new_state.keys() - assert new_state['input1'].dims == untouched_quantity.dims - assert np.allclose(new_state['input1'].values, 10.) - assert new_state['input1'].attrs['units'] == 'J' +class TestAdamsBashforthFirstOrder(TimesteppingBase, PrognosticBase): - def test_stepper_requires_input_for_stepped_quantity(self): - input_properties = {} - diagnostic_properties = {} - tendency_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm/s' - }, - } - diagnostic_output = {} - tendency_output = { - 'output1': np.ones([10]) * 2., - } - prognostic = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - stepper = self.timestepper_class(prognostic) - assert 'output1' in stepper.input_properties.keys() - assert stepper.input_properties['output1']['dims'] == ['dim1'] - assert units_are_compatible(stepper.input_properties['output1']['units'], 'm') + def timestepper_class(self, *args, **kwargs): + kwargs['order'] = 1 + return AdamsBashforth(*args, **kwargs) - def test_stepper_outputs_stepped_quantity(self): - input_properties = {} - diagnostic_properties = {} - tendency_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm/s' - }, - } - diagnostic_output = {} - tendency_output = { - 'output1': np.ones([10]) * 2., - } - prognostic = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - stepper = self.timestepper_class(prognostic) - assert 'output1' in stepper.output_properties.keys() - assert stepper.output_properties['output1']['dims'] == ['dim1'] - assert units_are_compatible(stepper.output_properties['output1']['units'], 'm') - def test_stepper_requires_input_for_input_quantity(self): - input_properties = { - 'input1': { - 'dims': ['dim1', 'dim2'], - 'units': 's', - } - } - diagnostic_properties = {} - tendency_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm/s' - }, - } - diagnostic_output = {} - tendency_output = { - 'output1': np.ones([10]) * 2., - } - prognostic = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - stepper = self.timestepper_class(prognostic) - assert 'input1' in stepper.input_properties.keys() - assert stepper.input_properties['input1']['dims'] == ['dim1', 'dim2'] - assert units_are_compatible(stepper.input_properties['input1']['units'], 's') - assert len(stepper.diagnostic_properties) == 0 +class TestAdamsBashforthFirstOrderImplicitPrognostic(TimesteppingBase, ImplicitPrognosticBase): - def test_stepper_gives_diagnostic_quantity(self): - input_properties = {} - diagnostic_properties = { - 'diag1': { - 'dims': ['dim2'], - 'units': '', - } - } - tendency_properties = { - } - diagnostic_output = {} - tendency_output = { - } - prognostic = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - stepper = self.timestepper_class( - prognostic, tendencies_in_diagnostics=True, name='component') - assert 'diag1' in stepper.diagnostic_properties.keys() - assert stepper.diagnostic_properties['diag1']['dims'] == ['dim2'] - assert units_are_compatible( - stepper.diagnostic_properties['diag1']['units'], '') - assert len(stepper.input_properties) == 0 - assert len(stepper.output_properties) == 0 + def timestepper_class(self, *args, **kwargs): + kwargs['order'] = 1 + return AdamsBashforth(*args, **kwargs) - def test_stepper_gives_diagnostic_tendency_quantity(self): - input_properties = { - 'input1': { - 'dims': ['dim1', 'dim2'], - 'units': 's', - } - } - diagnostic_properties = {} - tendency_properties = { - 'output1': { - 'dims': ['dim1'], - 'units': 'm/s' - }, - } - diagnostic_output = {} - tendency_output = { - 'output1': np.ones([10]) * 2., - } - prognostic = MockPrognostic( - input_properties, diagnostic_properties, tendency_properties, - diagnostic_output, tendency_output - ) - stepper = self.timestepper_class( - prognostic, tendencies_in_diagnostics=True) - tendency_name = 'output1_tendency_from_{}'.format(stepper.__class__.__name__) - assert tendency_name in stepper.diagnostic_properties.keys() - assert len(stepper.diagnostic_properties) == 1 - assert stepper.diagnostic_properties[tendency_name]['dims'] == ['dim1'] - assert units_are_compatible(stepper.input_properties['output1']['units'], 'm') - assert units_are_compatible(stepper.diagnostic_properties[tendency_name]['units'], 'm/s') + +class TestSSPRungeKuttaTwoStep(TimesteppingBase, PrognosticBase): + + def timestepper_class(self, *args, **kwargs): + kwargs['stages'] = 2 + return SSPRungeKutta(*args, **kwargs) -class TestSSPRungeKuttaTwoStep(TimesteppingBase): +class TestSSPRungeKuttaTwoStepImplicitPrognostic(TimesteppingBase, ImplicitPrognosticBase): def timestepper_class(self, *args, **kwargs): kwargs['stages'] = 2 return SSPRungeKutta(*args, **kwargs) -class TestSSPRungeKuttaThreeStep(TimesteppingBase): +class TestSSPRungeKuttaThreeStep(TimesteppingBase, PrognosticBase): + + def timestepper_class(self, *args, **kwargs): + kwargs['stages'] = 3 + return SSPRungeKutta(*args, **kwargs) + +class TestSSPRungeKuttaThreeStepImplicitPrognostic(TimesteppingBase, ImplicitPrognosticBase): def timestepper_class(self, *args, **kwargs): kwargs['stages'] = 3 return SSPRungeKutta(*args, **kwargs) -class TestLeapfrog(TimesteppingBase): +class TestLeapfrog(TimesteppingBase, PrognosticBase): + + timestepper_class = Leapfrog + + +class TestLeapfrogImplicitPrognostic(TimesteppingBase, ImplicitPrognosticBase): timestepper_class = Leapfrog -class TestAdamsBashforthSecondOrder(TimesteppingBase): +class TestAdamsBashforthSecondOrder(TimesteppingBase, PrognosticBase): + + def timestepper_class(self, *args, **kwargs): + kwargs['order'] = 2 + return AdamsBashforth(*args, **kwargs) + + +class TestAdamsBashforthSecondOrderImplicitPrognostic(TimesteppingBase, ImplicitPrognosticBase): def timestepper_class(self, *args, **kwargs): kwargs['order'] = 2 return AdamsBashforth(*args, **kwargs) -class TestAdamsBashforthThirdOrder(TimesteppingBase): +class TestAdamsBashforthThirdOrder(TimesteppingBase, PrognosticBase): + + def timestepper_class(self, *args, **kwargs): + kwargs['order'] = 3 + return AdamsBashforth(*args, **kwargs) + + +class TestAdamsBashforthThirdOrderImplicitPrognostic(TimesteppingBase, ImplicitPrognosticBase): def timestepper_class(self, *args, **kwargs): kwargs['order'] = 3 return AdamsBashforth(*args, **kwargs) -class TestAdamsBashforthFourthOrder(TimesteppingBase): +class TestAdamsBashforthFourthOrder(TimesteppingBase, PrognosticBase): def timestepper_class(self, *args, **kwargs): kwargs['order'] = 4 return AdamsBashforth(*args, **kwargs) - @mock.patch.object(MockEmptyPrognostic, '__call__') - def test_array_four_steps(self, mock_prognostic_call): - mock_prognostic_call.return_value = ( - {'air_temperature': np.ones((3, 3))*1.}, {}) - state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} - timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert same_list(state.keys(), ['time', 'air_temperature']) - assert (state['air_temperature'] == np.ones((3, 3))*273.).all() - assert same_list(new_state.keys(), ['time', 'air_temperature']) - assert (new_state['air_temperature'] == np.ones((3, 3))*274.).all() - state = new_state - diagnostics, new_state = time_stepper.__call__(new_state, timestep) - assert same_list(state.keys(), ['time', 'air_temperature']) - assert (state['air_temperature'] == np.ones((3, 3))*274.).all() - assert same_list(new_state.keys(), ['time', 'air_temperature']) - assert (new_state['air_temperature'] == np.ones((3, 3))*275.).all() - state = new_state - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert same_list(state.keys(), ['time', 'air_temperature']) - assert (state['air_temperature'] == np.ones((3, 3))*275.).all() - assert same_list(new_state.keys(), ['time', 'air_temperature']) - assert (new_state['air_temperature'] == np.ones((3, 3))*276.).all() - state = new_state - diagnostics, new_state = time_stepper.__call__(state, timestep) - assert same_list(state.keys(), ['time', 'air_temperature']) - assert (state['air_temperature'] == np.ones((3, 3))*276.).all() - assert same_list(new_state.keys(), ['time', 'air_temperature']) - assert (new_state['air_temperature'] == np.ones((3, 3))*277.).all() + +class TestAdamsBashforthFourthOrderImplicitPrognostic(TimesteppingBase, ImplicitPrognosticBase): + + def timestepper_class(self, *args, **kwargs): + kwargs['order'] = 4 + return AdamsBashforth(*args, **kwargs) @mock.patch.object(MockEmptyPrognostic, '__call__') From d01356db284cfe5e0c6b0f97f2a89bbfa2a32838 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 15 Jun 2018 14:28:37 -0700 Subject: [PATCH 69/98] Fixed warning test on Python 3, and flake8 --- sympl/_core/composite.py | 4 ++-- sympl/_core/timestepper.py | 2 -- tests/test_timestepping.py | 2 ++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 9ad6a3d..4f75d8f 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -170,7 +170,8 @@ def array_call(self, state): raise NotImplementedError() -class ImplicitPrognosticComposite(ComponentComposite, InputPropertiesCompositeMixin, +class ImplicitPrognosticComposite( + ComponentComposite, InputPropertiesCompositeMixin, DiagnosticPropertiesCompositeMixin, ImplicitPrognostic): component_class = (Prognostic, ImplicitPrognostic) @@ -243,7 +244,6 @@ def array_call(self, state): raise NotImplementedError() - class DiagnosticComposite( ComponentComposite, InputPropertiesCompositeMixin, DiagnosticPropertiesCompositeMixin, Diagnostic): diff --git a/sympl/_core/timestepper.py b/sympl/_core/timestepper.py index 9048010..88e1f06 100644 --- a/sympl/_core/timestepper.py +++ b/sympl/_core/timestepper.py @@ -141,9 +141,7 @@ def __init__(self, *args, **kwargs): self._tendencies_in_diagnostics = tendencies_in_diagnostics # warnings.simplefilter('always') if any(isinstance(a, ImplicitPrognostic) for a in args): - print('WARNING now') warnings.warn('ImplicitPrognostic') - print(warnings) self.prognostic = ImplicitPrognosticComposite(*args) @property diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index 44ead14..0991b04 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -416,6 +416,8 @@ def test_warn_on_implicitprognostic(self): prognostic = self.prognostic_class({}, {}, {}, {}, {}) with pytest.warns(UserWarning) as w: self.timestepper_class(prognostic) + if isinstance(w, Warning): + w = [w] print(list(i.message for i in w)) assert 'ImplicitPrognostic' in w[0].message From e12a56a8b9758b80fe8c95c09d935f547d525ce6 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 15 Jun 2018 14:34:54 -0700 Subject: [PATCH 70/98] Correcting userwarning test --- tests/test_timestepping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index 0991b04..634aa1c 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -416,7 +416,7 @@ def test_warn_on_implicitprognostic(self): prognostic = self.prognostic_class({}, {}, {}, {}, {}) with pytest.warns(UserWarning) as w: self.timestepper_class(prognostic) - if isinstance(w, Warning): + if isinstance(w, UserWarning): w = [w] print(list(i.message for i in w)) assert 'ImplicitPrognostic' in w[0].message From 1957fe328ab2b96900e4f48fbb3c23fd7c72e3aa Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 15 Jun 2018 16:06:08 -0700 Subject: [PATCH 71/98] Fix UserWarning test on ImplicitPrognostic in Python 3 --- tests/test_timestepping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index 634aa1c..6b1502b 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -419,7 +419,7 @@ def test_warn_on_implicitprognostic(self): if isinstance(w, UserWarning): w = [w] print(list(i.message for i in w)) - assert 'ImplicitPrognostic' in w[0].message + assert 'ImplicitPrognostic' in w[0].message.args[0] class TimesteppingBase(object): From 5978ed57ebae8de3299c882664f5eb4f1fc43de9 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 15 Jun 2018 16:08:47 -0700 Subject: [PATCH 72/98] Made ImplictPrognostic TimeStepper warning more verbose --- sympl/_core/timestepper.py | 5 ++++- tests/test_timestepping.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sympl/_core/timestepper.py b/sympl/_core/timestepper.py index 88e1f06..7008df0 100644 --- a/sympl/_core/timestepper.py +++ b/sympl/_core/timestepper.py @@ -141,7 +141,10 @@ def __init__(self, *args, **kwargs): self._tendencies_in_diagnostics = tendencies_in_diagnostics # warnings.simplefilter('always') if any(isinstance(a, ImplicitPrognostic) for a in args): - warnings.warn('ImplicitPrognostic') + warnings.warn( + 'Using an ImplicitPrognostic in sympl TimeStepper objects may ' + 'lead to scientifically invalid results. Make sure the component ' + 'follows the same numerical assumptions as the TimeStepper used.') self.prognostic = ImplicitPrognosticComposite(*args) @property diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index 6b1502b..7ab08ab 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -420,6 +420,7 @@ def test_warn_on_implicitprognostic(self): w = [w] print(list(i.message for i in w)) assert 'ImplicitPrognostic' in w[0].message.args[0] + assert 'scientifically invalid' in w[0].message.args[0] class TimesteppingBase(object): From af26ce7533373994a4945e25fd07907a0c55e5e3 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 15 Jun 2018 16:09:36 -0700 Subject: [PATCH 73/98] Began work on TracerPacker --- sympl/_core/tracers.py | 67 ++++++++++++++++++++++++++++++++++++++++++ tests/test_tracers.py | 49 ++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 sympl/_core/tracers.py create mode 100644 tests/test_tracers.py diff --git a/sympl/_core/tracers.py b/sympl/_core/tracers.py new file mode 100644 index 0000000..8ebc4c6 --- /dev/null +++ b/sympl/_core/tracers.py @@ -0,0 +1,67 @@ +from .exceptions import InvalidPropertyDictError +import numpy as np + +_tracer_unit_dict = {} + + +def register_tracer(name, units): + _tracer_unit_dict[name] = units + + +def get_quantity_dims(tracer_dims): + if 'tracer' not in tracer_dims: + raise ValueError("Tracer dims must include a dimension named 'tracer'") + quantity_dims = list(tracer_dims) + quantity_dims.pop('tracer') + return tuple(quantity_dims) + + +class TracerPacker(object): + + def __init__(self, tracer_dims, component=None): + self.tracer_names = [] + self._tracer_dims = tuple(tracer_dims) + self._tracer_quantity_dims = get_quantity_dims(tracer_dims) + if component is not None: + for name, properties in component.input_properties.items(): + if properties.get('tracer'): + self._ensure_tracer_quantity_dims(properties['dims']) + self.tracer_names.append(name) + + def _ensure_tracer_quantity_dims(self, dims): + if tuple(self._tracer_quantity_dims) != tuple(dims): + raise InvalidPropertyDictError( + 'Tracers have conflicting dims {} and {}'.format( + self._tracer_quantity_dims, dims) + ) + + def register_tracers(self, unit_dict, input_properties): + for name, units in unit_dict.items(): + if name not in input_properties.keys(): + input_properties[name] = { + 'dims': self._tracer_quantity_dims, + 'units': units, + 'tracer': True, + } + + @property + def _tracer_index(self): + return self._tracer_dims.index('tracer') + + def pack_tracers(self, raw_state): + shape = list(raw_state[self.tracer_names[0]].shape) + shape.insert(self._tracer_index, len(self.tracer_names)) + array = np.empty(shape, dtype=np.float64) + for i, name in enumerate(self.tracer_names): + tracer_slice = [slice(0, d) for d in shape] + tracer_slice[self._tracer_index] = i + array[tracer_slice] = raw_state[name] + return array + + def unpack_tracers(self, tracer_array): + return_state = {} + for i, name in enumerate(self.tracer_names): + tracer_slice = [slice(0, d) for d in tracer_array.shape] + tracer_slice[self._tracer_index] = i + return_state[name] = tracer_array[tracer_slice] + return return_state diff --git a/tests/test_tracers.py b/tests/test_tracers.py new file mode 100644 index 0000000..c58a6ac --- /dev/null +++ b/tests/test_tracers.py @@ -0,0 +1,49 @@ +from sympl._core.tracers import TracerPacker, register_tracer +import unittest +import numpy as np + +class TracerPackerTests(unittest.TestCase): + + def test_packs_no_tracers(self): + dims = ['tracer', '*'] + packer = TracerPacker(dims) + packed = packer.pack({}) + assert isinstance(packed, np.ndarray) + assert packed.shape == [0, 0] + + def test_unpacks_no_tracers(self): + dims = ['tracer', '*'] + packer = TracerPacker(dims) + unpacked = packer.unpack({}) + assert isinstance(unpacked, dict) + assert len(unpacked) == 0 + + def test_unpacks_no_tracers_with_arrays_input(self): + dims = ['tracer', '*'] + packer = TracerPacker(dims) + unpacked = packer.unpack({'air_temperature': np.zeros((5,))}) + assert isinstance(unpacked, dict) + assert len(unpacked) == 0 + + def test_packs_one_tracer(self): + np.random.seed(0) + dims = ['tracer', '*'] + register_tracer('tracer1', 'g/m^3') + raw_state = {'tracer1': np.random.randn(5)} + packer = TracerPacker(dims, raw_state) + packed = packer.pack(raw_state) + assert isinstance(packed, np.ndarray) + assert packed.shape == [1, 5] + assert np.all(packed[0, :] == raw_state['tracer1']) + + def test_packs_one_tracer_registered_after_init(self): + np.random.seed(0) + dims = ['tracer', '*'] + raw_state = {'tracer1': np.random.randn(5)} + packer = TracerPacker(dims, raw_state) + register_tracer('tracer1', 'g/m^3') + packed = packer.pack(raw_state) + assert isinstance(packed, np.ndarray) + assert packed.shape == [1, 5] + assert np.all(packed[0, :] == raw_state['tracer1']) + From d41a3f3840294741627280287d0136fd88f8a923 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 15 Jun 2018 17:50:08 -0700 Subject: [PATCH 74/98] More work on tracer packing --- sympl/__init__.py | 2 + sympl/_core/tracers.py | 19 +++++-- tests/test_tracers.py | 118 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 129 insertions(+), 10 deletions(-) diff --git a/sympl/__init__.py b/sympl/__init__.py index 65d8cd0..36c1f5d 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -14,6 +14,7 @@ from ._core.constants import ( get_constant, set_constant, set_condensible_name, reset_constants, get_constants_string) +from ._core.tracers import register_tracer, get_tracer_unit_dict from ._core.util import ( ensure_no_shared_keys, get_numpy_array, jit, @@ -44,6 +45,7 @@ get_constants_string, TimeDifferencingWrapper, ensure_no_shared_keys, get_numpy_array, jit, + register_tracer, get_tracer_unit_dict, restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, initialize_numpy_arrays_with_properties, diff --git a/sympl/_core/tracers.py b/sympl/_core/tracers.py index 8ebc4c6..0faa94e 100644 --- a/sympl/_core/tracers.py +++ b/sympl/_core/tracers.py @@ -4,21 +4,32 @@ _tracer_unit_dict = {} +def clear_tracer_unit_dict(): + while len(_tracer_unit_dict) > 0: + _tracer_unit_dict.pop() + + def register_tracer(name, units): _tracer_unit_dict[name] = units +def get_tracer_unit_dict(): + return_dict = {} + return_dict.update(_tracer_unit_dict) + return return_dict + + def get_quantity_dims(tracer_dims): if 'tracer' not in tracer_dims: raise ValueError("Tracer dims must include a dimension named 'tracer'") quantity_dims = list(tracer_dims) - quantity_dims.pop('tracer') + quantity_dims.remove('tracer') return tuple(quantity_dims) class TracerPacker(object): - def __init__(self, tracer_dims, component=None): + def __init__(self, component, tracer_dims): self.tracer_names = [] self._tracer_dims = tuple(tracer_dims) self._tracer_quantity_dims = get_quantity_dims(tracer_dims) @@ -48,7 +59,7 @@ def register_tracers(self, unit_dict, input_properties): def _tracer_index(self): return self._tracer_dims.index('tracer') - def pack_tracers(self, raw_state): + def pack(self, raw_state): shape = list(raw_state[self.tracer_names[0]].shape) shape.insert(self._tracer_index, len(self.tracer_names)) array = np.empty(shape, dtype=np.float64) @@ -58,7 +69,7 @@ def pack_tracers(self, raw_state): array[tracer_slice] = raw_state[name] return array - def unpack_tracers(self, tracer_array): + def unpack(self, tracer_array): return_state = {} for i, name in enumerate(self.tracer_names): tracer_slice = [slice(0, d) for d in tracer_array.shape] diff --git a/tests/test_tracers.py b/tests/test_tracers.py index c58a6ac..e9d59ef 100644 --- a/tests/test_tracers.py +++ b/tests/test_tracers.py @@ -1,26 +1,110 @@ -from sympl._core.tracers import TracerPacker, register_tracer +from sympl._core.tracers import TracerPacker, clear_tracer_unit_dict +from sympl import Prognostic, register_tracer, get_tracer_unit_dict import unittest import numpy as np +import pytest + + +class MockPrognostic(Prognostic): + + input_properties = None + diagnostic_properties = None + tendency_properties = None + + def __init__(self, **kwargs): + self.input_properties = {} + self.diagnostic_properties = {} + self.tendency_properties = {} + self.diagnostic_output = {} + self.tendency_output = {} + self.times_called = 0 + self.state_given = None + super(MockPrognostic, self).__init__(**kwargs) + + def array_call(self, state): + self.times_called += 1 + self.state_given = state + return self.tendency_output, self.diagnostic_output + + +""" +On init and tracer registration, should update input properties of its +component. + +On pack, should pack all tracers not already present in input_properties. + +On unpack, should unpack all tracers not present in input_properties + +Keep track of internal representation of tracer order. + On adding new tracer, add it to the end. + Allow global tracer order to be set. +""" + + +class RegisterTracerTests(unittest.TestCase): + + def setUp(self): + clear_tracer_unit_dict() + + def test_initially_empty(self): + assert len(get_tracer_unit_dict()) == 0 + + def test_register_one_tracer(self): + register_tracer('tracer1', 'm') + d = get_tracer_unit_dict() + assert len(d) == 1 + assert 'tracer1' in d + assert d['tracer1'] == 'unit1' + + def test_register_two_tracers(self): + register_tracer('tracer1', 'm') + register_tracer('tracer2', 'degK') + d = get_tracer_unit_dict() + assert len(d) == 2 + assert 'tracer1' in d + assert 'tracer2' in d + assert d['tracer1'] == 'm' + assert d['tracer2'] == 'degK' + + def test_reregister_tracer(self): + register_tracer('tracer1', 'm') + register_tracer('tracer1', 'm') + d = get_tracer_unit_dict() + assert len(d) == 1 + assert 'tracer1' in d + assert d['tracer1'] == 'm' + + def test_reregister_tracer_different_units(self): + register_tracer('tracer1', 'm') + with self.assertRaises(ValueError): + register_tracer('tracer1', 'degK') + class TracerPackerTests(unittest.TestCase): + def setUp(self): + self.prognostic = MockPrognostic() + + def tearDown(self): + self.diagnostic = None + def test_packs_no_tracers(self): dims = ['tracer', '*'] - packer = TracerPacker(dims) + packer = TracerPacker(self.prognostic, dims) packed = packer.pack({}) assert isinstance(packed, np.ndarray) assert packed.shape == [0, 0] def test_unpacks_no_tracers(self): dims = ['tracer', '*'] - packer = TracerPacker(dims) + packer = TracerPacker(self.prognostic, dims) unpacked = packer.unpack({}) assert isinstance(unpacked, dict) assert len(unpacked) == 0 def test_unpacks_no_tracers_with_arrays_input(self): dims = ['tracer', '*'] - packer = TracerPacker(dims) + packer = TracerPacker(self.prognostic, dims) unpacked = packer.unpack({'air_temperature': np.zeros((5,))}) assert isinstance(unpacked, dict) assert len(unpacked) == 0 @@ -30,17 +114,39 @@ def test_packs_one_tracer(self): dims = ['tracer', '*'] register_tracer('tracer1', 'g/m^3') raw_state = {'tracer1': np.random.randn(5)} - packer = TracerPacker(dims, raw_state) + packer = TracerPacker(self.prognostic, dims) packed = packer.pack(raw_state) assert isinstance(packed, np.ndarray) assert packed.shape == [1, 5] assert np.all(packed[0, :] == raw_state['tracer1']) + def test_packs_updates_properties(self): + np.random.seed(0) + dims = ['tracer', '*'] + register_tracer('tracer1', 'g/m^3') + packer = TracerPacker(self.prognostic, dims) + assert 'tracer1' in self.prognostic.input_properties + assert tuple(self.prognostic.input_propertes['tracer1']['dims']) == ('*',) + assert self.prognostic.input_properties['units'] == 'g/m^3' + assert len(self.prognostic.input_properties) == 1 + + def test_packs_updates_properties_after_init(self): + np.random.seed(0) + dims = ['tracer', '*'] + packer = TracerPacker(self.prognostic, dims) + assert len(self.prognostic.input_properties) == 0 + register_tracer('tracer1', 'g/m^3') + assert 'tracer1' in self.prognostic.input_properties + assert tuple( + self.prognostic.input_propertes['tracer1']['dims']) == ('*',) + assert self.prognostic.input_properties['units'] == 'g/m^3' + assert len(self.prognostic.input_properties) == 1 + def test_packs_one_tracer_registered_after_init(self): np.random.seed(0) dims = ['tracer', '*'] raw_state = {'tracer1': np.random.randn(5)} - packer = TracerPacker(dims, raw_state) + packer = TracerPacker(self.prognostic, dims) register_tracer('tracer1', 'g/m^3') packed = packer.pack(raw_state) assert isinstance(packed, np.ndarray) From aaa96f7c7be2124fc9abac7b37c72e4400db00e0 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 18 Jun 2018 11:58:52 -0700 Subject: [PATCH 75/98] Removed test for warning on timestepping ImplicitPrognostic --- tests/test_timestepping.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index 7ab08ab..003b15c 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -7,8 +7,6 @@ import numpy as np import warnings -warnings.filterwarnings('always', 'ImplicitPrognostic') - def same_list(list1, list2): return (len(list1) == len(list2) and all( @@ -412,16 +410,6 @@ class ImplicitPrognosticBase(PrognosticBase): prognostic_class = MockImplicitPrognostic - def test_warn_on_implicitprognostic(self): - prognostic = self.prognostic_class({}, {}, {}, {}, {}) - with pytest.warns(UserWarning) as w: - self.timestepper_class(prognostic) - if isinstance(w, UserWarning): - w = [w] - print(list(i.message for i in w)) - assert 'ImplicitPrognostic' in w[0].message.args[0] - assert 'scientifically invalid' in w[0].message.args[0] - class TimesteppingBase(object): From 552cf6f2de43ecc3ab76f996151869d54b5208c1 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 18 Jun 2018 13:43:16 -0700 Subject: [PATCH 76/98] Added unit helper functions to public API --- HISTORY.rst | 2 ++ sympl/__init__.py | 4 +++- sympl/_core/units.py | 30 ++++++++++++++++++++++++++++++ tests/test_units.py | 24 ++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 tests/test_units.py diff --git a/HISTORY.rst b/HISTORY.rst index f236c33..5ff3945 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -47,6 +47,8 @@ Latest * Fixed bug where degrees Celcius or Fahrenheit could not be used as units on inputs because it would lead to an error. * Added combine_component_properties as a public function. +* Added some unit helper functions (units_are_same, units_are_compatible, + is_valid_unit) to public API. Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/__init__.py b/sympl/__init__.py index 36c1f5d..334647a 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -14,13 +14,14 @@ from ._core.constants import ( get_constant, set_constant, set_condensible_name, reset_constants, get_constants_string) -from ._core.tracers import register_tracer, get_tracer_unit_dict +from ._core.tracers import register_tracer, get_tracer_unit_dict, get_tracer_names from ._core.util import ( ensure_no_shared_keys, get_numpy_array, jit, restore_dimensions, get_component_aliases, combine_component_properties) +from ._core.units import units_are_same, units_are_compatible, is_valid_unit from ._core.state import ( get_numpy_arrays_with_properties, restore_data_arrays_with_properties, @@ -40,6 +41,7 @@ InvalidStateError, SharedKeyError, DependencyError, InvalidPropertyDictError, ComponentExtraOutputError, ComponentMissingOutputError, + units_are_same, units_are_compatible, is_valid_unit, DataArray, get_constant, set_constant, set_condensible_name, reset_constants, get_constants_string, TimeDifferencingWrapper, diff --git a/sympl/_core/units.py b/sympl/_core/units.py index 29358ab..d2977e8 100644 --- a/sympl/_core/units.py +++ b/sympl/_core/units.py @@ -20,6 +20,19 @@ def __call__(self, input_string, **kwargs): def units_are_compatible(unit1, unit2): + """ + Determine whether a unit can be converted to another unit. + + Parameters + ---------- + unit1 : str + unit2 : str + + Returns + ------- + units_are_compatible : bool + True if the first unit can be converted to the second unit. + """ try: unit_registry(unit1).to(unit2) return True @@ -27,6 +40,23 @@ def units_are_compatible(unit1, unit2): return False +def units_are_same(unit1, unit2): + """ + Compare two unit strings for equality. + + Parameters + ---------- + unit1 : str + unit2 : str + + Returns + ------- + units_are_same : bool + True if the two input unit strings represent the same unit. + """ + return unit_registry(unit1) == unit_registry(unit2) + + def clean_units(unit_string): return str(unit_registry(unit_string).to_base_units().units) diff --git a/tests/test_units.py b/tests/test_units.py new file mode 100644 index 0000000..6ae08c0 --- /dev/null +++ b/tests/test_units.py @@ -0,0 +1,24 @@ +from sympl import units_are_same, units_are_compatible, is_valid_unit + + +def test_is_valid_unit_meters(): + assert is_valid_unit('m') + assert is_valid_unit('meter') + assert is_valid_unit('meters') + + +def test_units_are_compatible_meters(): + assert units_are_compatible('m', 'km') + assert units_are_compatible('kilometers', 'cm') + assert not units_are_compatible('m', 'm/s') + + +def test_units_are_same_meters(): + assert units_are_same('m', 'meter') + assert units_are_same('meters', 'm') + assert units_are_same('kilometers', 'km') + + +def test_is_valid_unit_invalid_values(): + assert not is_valid_unit('george') + assert not is_valid_unit('boop') From 9454a644558fc43d280b0e01ac83587183133246 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 18 Jun 2018 15:21:12 -0700 Subject: [PATCH 77/98] Finished tracer handling functionality for base classes --- HISTORY.rst | 1 + sympl/_core/base_components.py | 46 +++ sympl/_core/tracers.py | 120 +++++++- tests/test_tracers.py | 543 +++++++++++++++++++++++++++++++-- 4 files changed, 672 insertions(+), 38 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 5ff3945..d30a79c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -49,6 +49,7 @@ Latest * Added combine_component_properties as a public function. * Added some unit helper functions (units_are_same, units_are_compatible, is_valid_unit) to public API. +* Added tracer-handling funcitonality to component base classes. Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 823953a..c02c40f 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -6,6 +6,7 @@ ComponentMissingOutputError, InvalidStateError) from six import add_metaclass from .units import units_are_compatible +from .tracers import TracerPacker try: from inspect import getfullargspec as getargspec except ImportError: @@ -439,6 +440,8 @@ class Implicit(object): time_unit_name = 's' time_unit_timedelta = timedelta(seconds=1) + uses_tracers = False + tracer_dims = None @abc.abstractproperty def input_properties(self): @@ -501,6 +504,12 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): self._diagnostic_checker.set_ignored_diagnostics( self._insert_tendency_properties()) self.__initialized = True + if self.uses_tracers: + if self.tracer_dims is None: + raise ValueError( + 'Component of type {} must specify tracer_dims property ' + 'when uses_tracers=True'.format(self.__class__.__name__)) + self._tracer_packer = TracerPacker(self, self.tracer_dims) super(Implicit, self).__init__() def _insert_tendency_properties(self): @@ -599,8 +608,15 @@ def __call__(self, state, timestep): self.__check_self_is_initialized() self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + if self.uses_tracers: + raw_state['tracers'] = self._tracer_packer.pack(raw_state) + for name in self._tracer_packer.tracer_names: + raw_state.pop(name) raw_state['time'] = state['time'] raw_diagnostics, raw_new_state = self.array_call(raw_state, timestep) + if self.uses_tracers: + raw_new_state.update(self._tracer_packer.unpack(raw_new_state['tracers'])) + raw_new_state.pop('tracers') self._diagnostic_checker.check_diagnostics(raw_diagnostics) self._output_checker.check_outputs(raw_new_state) if self.tendencies_in_diagnostics: @@ -688,6 +704,8 @@ def diagnostic_properties(self): return {} name = None + uses_tracers = False + tracer_tendency_time_unit = 's' def __str__(self): return ( @@ -737,6 +755,12 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): self._diagnostic_checker.set_ignored_diagnostics(self._added_diagnostic_names) else: self._added_diagnostic_names = [] + if self.uses_tracers: + if self.tracer_dims is None: + raise ValueError( + 'Component of type {} must specify tracer_dims property ' + 'when uses_tracers=True'.format(self.__class__.__name__)) + self._tracer_packer = TracerPacker(self, self.tracer_dims) self.__initialized = True super(Prognostic, self).__init__() @@ -808,8 +832,15 @@ def __call__(self, state): self.__check_self_is_initialized() self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + if self.uses_tracers: + raw_state['tracers'] = self._tracer_packer.pack(raw_state) + for name in self._tracer_packer.tracer_names: + raw_state.pop(name) raw_state['time'] = state['time'] raw_tendencies, raw_diagnostics = self.array_call(raw_state) + if self.uses_tracers: + raw_tendencies.update(self._tracer_packer.unpack(raw_tendencies['tracers'])) + raw_tendencies.pop('tracers') self._tendency_checker.check_tendencies(raw_tendencies) self._diagnostic_checker.check_diagnostics(raw_diagnostics) tendencies = restore_data_arrays_with_properties( @@ -895,6 +926,8 @@ def diagnostic_properties(self): return {} name = None + uses_tracers = False + tracer_tendency_time_unit = 's' def __str__(self): return ( @@ -941,6 +974,12 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): if self.tendencies_in_diagnostics: self._added_diagnostic_names = self._insert_tendency_properties() self._diagnostic_checker.set_ignored_diagnostics(self._added_diagnostic_names) + if self.uses_tracers: + if self.tracer_dims is None: + raise ValueError( + 'Component of type {} must specify tracer_dims property ' + 'when uses_tracers=True'.format(self.__class__.__name__)) + self._tracer_packer = TracerPacker(self, self.tracer_dims) self.__initialized = True super(ImplicitPrognostic, self).__init__() @@ -1014,8 +1053,15 @@ def __call__(self, state, timestep): self.__check_self_is_initialized() self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) + if self.uses_tracers: + raw_state['tracers'] = self._tracer_packer.pack(raw_state) + for name in self._tracer_packer.tracer_names: + raw_state.pop(name) raw_state['time'] = state['time'] raw_tendencies, raw_diagnostics = self.array_call(raw_state, timestep) + if self.uses_tracers: + raw_tendencies.update(self._tracer_packer.unpack(raw_tendencies['tracers'])) + raw_tendencies.pop('tracers') self._tendency_checker.check_tendencies(raw_tendencies) self._diagnostic_checker.check_diagnostics(raw_diagnostics) tendencies = restore_data_arrays_with_properties( diff --git a/sympl/_core/tracers.py b/sympl/_core/tracers.py index 0faa94e..3666db6 100644 --- a/sympl/_core/tracers.py +++ b/sympl/_core/tracers.py @@ -1,24 +1,70 @@ from .exceptions import InvalidPropertyDictError import numpy as np +from .units import units_are_same _tracer_unit_dict = {} +_tracer_names = [] +_packers = set() -def clear_tracer_unit_dict(): +def clear_tracers(): + global _tracer_names while len(_tracer_unit_dict) > 0: - _tracer_unit_dict.pop() + _tracer_unit_dict.popitem() + while len(_tracer_names) > 0: + _tracer_names.pop() + + +def clear_packers(): + while len(_packers) > 0: + _packers.pop() def register_tracer(name, units): + """ + Parameters + ---------- + name : str + Quantity name to register as a tracer. + units : str + Unit string of that quantity. + """ + if name in _tracer_unit_dict and not units_are_same(_tracer_unit_dict[name], units): + raise ValueError( + 'Tracer {} is already registered with units {} which are different ' + 'from input units {}'.format( + name, _tracer_unit_dict[name], units + ) + ) _tracer_unit_dict[name] = units + _tracer_names.append(name) + for packer in _packers: + packer.insert_tracer_to_properties(name, units) def get_tracer_unit_dict(): + """ + Returns + ------- + unit_dict : dict + A dictionary whose keys are tracer quantity names as str and values are + units of those quantities as str. + """ return_dict = {} return_dict.update(_tracer_unit_dict) return return_dict +def get_tracer_names(): + """ + Returns + ------- + tracer_names : tuple of str + Tracer names in the order that they will appear in tracer arrays.\ + """ + return tuple(_tracer_names) + + def get_quantity_dims(tracer_dims): if 'tracer' not in tracer_dims: raise ValueError("Tracer dims must include a dimension named 'tracer'") @@ -30,14 +76,57 @@ def get_quantity_dims(tracer_dims): class TracerPacker(object): def __init__(self, component, tracer_dims): - self.tracer_names = [] self._tracer_dims = tuple(tracer_dims) self._tracer_quantity_dims = get_quantity_dims(tracer_dims) - if component is not None: - for name, properties in component.input_properties.items(): - if properties.get('tracer'): - self._ensure_tracer_quantity_dims(properties['dims']) - self.tracer_names.append(name) + if hasattr(component, 'tendency_properties') or hasattr(component, 'output_properties'): + self.component = component + else: + raise TypeError( + 'Expected a component object subclassing type Implicit, ' + 'ImplicitPrognostic, or Prognostic but received component of ' + 'type {}'.format(component.__class__.__name__)) + for name, units in _tracer_unit_dict.items(): + self.insert_tracer_to_properties(name, units) + _packers.add(self) + + def insert_tracer_to_properties(self, name, units): + self._insert_tracer_to_input_properties(name, units) + if hasattr(self.component, 'tendency_properties'): + self._insert_tracer_to_tendency_properties(name, units) + elif hasattr(self.component, 'output_properties'): + self._insert_tracer_to_output_properties(name, units) + + def _insert_tracer_to_input_properties(self, name, units): + if name not in self.component.input_properties: + self.component.input_properties[name] = { + 'dims': self._tracer_quantity_dims, + 'units': units, + 'tracer': True, + } + + def _insert_tracer_to_output_properties(self, name, units): + if name not in self.component.output_properties: + self.component.output_properties[name] = { + 'dims': self._tracer_quantity_dims, + 'units': units, + 'tracer': True, + } + + def _insert_tracer_to_tendency_properties(self, name, units): + time_unit = getattr(self.component, 'tracer_tendency_time_unit', 's') + if name not in self.component.tendency_properties: + self.component.tendency_properties[name] = { + 'dims': self._tracer_quantity_dims, + 'units': '{} {}^-1'.format(units, time_unit), + 'tracer': True, + } + + def remove_tracer_from_properties(self, name): + if self.is_tracer(name): + self.component.pop(name) + + def is_tracer(self, tracer_name): + return self.component.input_properties.get(tracer_name, {}).get('tracer', False) def _ensure_tracer_quantity_dims(self, dims): if tuple(self._tracer_quantity_dims) != tuple(dims): @@ -55,13 +144,24 @@ def register_tracers(self, unit_dict, input_properties): 'tracer': True, } + @property + def tracer_names(self): + return_list = [] + for name in _tracer_names: + if self.is_tracer(name): + return_list.append(name) + return tuple(return_list) + @property def _tracer_index(self): return self._tracer_dims.index('tracer') def pack(self, raw_state): - shape = list(raw_state[self.tracer_names[0]].shape) - shape.insert(self._tracer_index, len(self.tracer_names)) + if len(self.tracer_names) == 0: + shape = [0 for dim in self._tracer_dims] + else: + shape = list(raw_state[self.tracer_names[0]].shape) + shape.insert(self._tracer_index, len(self.tracer_names)) array = np.empty(shape, dtype=np.float64) for i, name in enumerate(self.tracer_names): tracer_slice = [slice(0, d) for d in shape] diff --git a/tests/test_tracers.py b/tests/test_tracers.py index e9d59ef..5ff4c6f 100644 --- a/tests/test_tracers.py +++ b/tests/test_tracers.py @@ -1,8 +1,12 @@ -from sympl._core.tracers import TracerPacker, clear_tracer_unit_dict -from sympl import Prognostic, register_tracer, get_tracer_unit_dict +from sympl._core.tracers import TracerPacker, clear_tracers, clear_packers +from sympl import ( + Prognostic, Implicit, Diagnostic, ImplicitPrognostic, register_tracer, + get_tracer_unit_dict, units_are_compatible, DataArray +) import unittest import numpy as np import pytest +from datetime import timedelta class MockPrognostic(Prognostic): @@ -27,6 +31,158 @@ def array_call(self, state): return self.tendency_output, self.diagnostic_output +class MockTracerPrognostic(Prognostic): + + input_properties = None + diagnostic_properties = None + tendency_properties = None + + uses_tracers = True + tracer_dims = ('tracer', '*') + + def __init__(self, **kwargs): + self.input_properties = {} + self.diagnostic_properties = {} + self.tendency_properties = {} + self.diagnostic_output = {} + self.times_called = 0 + self.state_given = None + super(MockTracerPrognostic, self).__init__(**kwargs) + + def array_call(self, state): + self.times_called += 1 + self.state_given = state + return_state = {} + return_state.update(state) + return_state.pop('time') + return return_state, self.diagnostic_output + + +class MockImplicitPrognostic(ImplicitPrognostic): + + input_properties = None + diagnostic_properties = None + tendency_properties = None + + def __init__( self, **kwargs): + self.input_properties = {} + self.diagnostic_properties = {} + self.tendency_properties = {} + self.diagnostic_output = {} + self.tendency_output = {} + self.times_called = 0 + self.state_given = None + self.timestep_given = None + super(MockImplicitPrognostic, self).__init__(**kwargs) + + def array_call(self, state, timestep): + self.times_called += 1 + self.state_given = state + self.timestep_given = timestep + return self.tendency_output, self.diagnostic_output + + +class MockTracerImplicitPrognostic(ImplicitPrognostic): + + input_properties = None + diagnostic_properties = None + tendency_properties = None + + uses_tracers = True + tracer_dims = ('tracer', '*') + + def __init__( self, **kwargs): + self.input_properties = {} + self.diagnostic_properties = {} + self.tendency_properties = {} + self.diagnostic_output = {} + self.times_called = 0 + self.state_given = None + self.timestep_given = None + super(MockTracerImplicitPrognostic, self).__init__(**kwargs) + + def array_call(self, state, timestep): + self.times_called += 1 + self.state_given = state + self.timestep_given = timestep + return_state = {} + return_state.update(state) + return_state.pop('time') + return return_state, self.diagnostic_output + + +class MockDiagnostic(Diagnostic): + + input_properties = None + diagnostic_properties = None + + def __init__(self, **kwargs): + self.input_properties = {} + self.diagnostic_properties = {} + self.diagnostic_output = {} + self.times_called = 0 + self.state_given = None + super(MockDiagnostic, self).__init__(**kwargs) + + def array_call(self, state): + self.times_called += 1 + self.state_given = state + return self.diagnostic_output + + +class MockImplicit(Implicit): + + input_properties = None + diagnostic_properties = None + output_properties = None + + def __init__(self, **kwargs): + self.input_properties = {} + self.diagnostic_properties = {} + self.output_properties = {} + self.diagnostic_output = {} + self.state_output = {} + self.times_called = 0 + self.state_given = None + self.timestep_given = None + super(MockImplicit, self).__init__(**kwargs) + + def array_call(self, state, timestep): + self.times_called += 1 + self.state_given = state + self.timestep_given = timestep + return self.diagnostic_output, self.state_output + + +class MockTracerImplicit(Implicit): + + input_properties = None + diagnostic_properties = None + output_properties = None + + uses_tracers = True + tracer_dims = ('tracer', '*') + + def __init__(self, **kwargs): + self.input_properties = {} + self.diagnostic_properties = {} + self.output_properties = {} + self.diagnostic_output = {} + self.state_output = {} + self.times_called = 0 + self.state_given = None + self.timestep_given = None + super(MockTracerImplicit, self).__init__(**kwargs) + + def array_call(self, state, timestep): + self.times_called += 1 + self.state_given = state + self.timestep_given = timestep + return_state = {} + return_state.update(state) + return_state.pop('time') + return self.diagnostic_output, return_state + """ On init and tracer registration, should update input properties of its component. @@ -44,7 +200,10 @@ def array_call(self, state): class RegisterTracerTests(unittest.TestCase): def setUp(self): - clear_tracer_unit_dict() + clear_tracers() + + def tearDown(self): + clear_tracers() def test_initially_empty(self): assert len(get_tracer_unit_dict()) == 0 @@ -54,7 +213,7 @@ def test_register_one_tracer(self): d = get_tracer_unit_dict() assert len(d) == 1 assert 'tracer1' in d - assert d['tracer1'] == 'unit1' + assert d['tracer1'] == 'm' def test_register_two_tracers(self): register_tracer('tracer1', 'm') @@ -80,31 +239,33 @@ def test_reregister_tracer_different_units(self): register_tracer('tracer1', 'degK') -class TracerPackerTests(unittest.TestCase): +class TracerPackerBase(object): def setUp(self): - self.prognostic = MockPrognostic() + clear_tracers() + clear_packers() def tearDown(self): - self.diagnostic = None + clear_tracers() + clear_packers() def test_packs_no_tracers(self): dims = ['tracer', '*'] - packer = TracerPacker(self.prognostic, dims) + packer = TracerPacker(self.component, dims) packed = packer.pack({}) assert isinstance(packed, np.ndarray) - assert packed.shape == [0, 0] + assert packed.shape == (0, 0) def test_unpacks_no_tracers(self): dims = ['tracer', '*'] - packer = TracerPacker(self.prognostic, dims) + packer = TracerPacker(self.component, dims) unpacked = packer.unpack({}) assert isinstance(unpacked, dict) assert len(unpacked) == 0 def test_unpacks_no_tracers_with_arrays_input(self): dims = ['tracer', '*'] - packer = TracerPacker(self.prognostic, dims) + packer = TracerPacker(self.component, dims) unpacked = packer.unpack({'air_temperature': np.zeros((5,))}) assert isinstance(unpacked, dict) assert len(unpacked) == 0 @@ -114,42 +275,368 @@ def test_packs_one_tracer(self): dims = ['tracer', '*'] register_tracer('tracer1', 'g/m^3') raw_state = {'tracer1': np.random.randn(5)} - packer = TracerPacker(self.prognostic, dims) + packer = TracerPacker(self.component, dims) packed = packer.pack(raw_state) assert isinstance(packed, np.ndarray) - assert packed.shape == [1, 5] + assert packed.shape == (1, 5) assert np.all(packed[0, :] == raw_state['tracer1']) - def test_packs_updates_properties(self): + def test_packs_one_3d_tracer(self): + np.random.seed(0) + dims = ['tracer', 'latitude', 'longitude', 'mid_levels'] + register_tracer('tracer1', 'g/m^3') + raw_state = {'tracer1': np.random.randn(2, 3, 4)} + packer = TracerPacker(self.component, dims) + packed = packer.pack(raw_state) + assert isinstance(packed, np.ndarray) + assert packed.shape == (1, 2, 3, 4) + assert np.all(packed[0, :, :, :] == raw_state['tracer1']) + + def test_packs_updates_input_properties(self): np.random.seed(0) dims = ['tracer', '*'] register_tracer('tracer1', 'g/m^3') - packer = TracerPacker(self.prognostic, dims) - assert 'tracer1' in self.prognostic.input_properties - assert tuple(self.prognostic.input_propertes['tracer1']['dims']) == ('*',) - assert self.prognostic.input_properties['units'] == 'g/m^3' - assert len(self.prognostic.input_properties) == 1 + packer = TracerPacker(self.component, dims) + assert 'tracer1' in self.component.input_properties + assert tuple(self.component.input_properties['tracer1']['dims']) == ('*',) + assert self.component.input_properties['tracer1']['units'] == 'g/m^3' + assert len(self.component.input_properties) == 1 - def test_packs_updates_properties_after_init(self): + def test_packs_updates_input_properties_after_init(self): np.random.seed(0) dims = ['tracer', '*'] - packer = TracerPacker(self.prognostic, dims) - assert len(self.prognostic.input_properties) == 0 + packer = TracerPacker(self.component, dims) + assert len(self.component.input_properties) == 0 register_tracer('tracer1', 'g/m^3') - assert 'tracer1' in self.prognostic.input_properties + assert 'tracer1' in self.component.input_properties assert tuple( - self.prognostic.input_propertes['tracer1']['dims']) == ('*',) - assert self.prognostic.input_properties['units'] == 'g/m^3' - assert len(self.prognostic.input_properties) == 1 + self.component.input_properties['tracer1']['dims']) == ('*',) + assert self.component.input_properties['tracer1']['units'] == 'g/m^3' + assert len(self.component.input_properties) == 1 def test_packs_one_tracer_registered_after_init(self): np.random.seed(0) dims = ['tracer', '*'] raw_state = {'tracer1': np.random.randn(5)} - packer = TracerPacker(self.prognostic, dims) + packer = TracerPacker(self.component, dims) register_tracer('tracer1', 'g/m^3') packed = packer.pack(raw_state) assert isinstance(packed, np.ndarray) - assert packed.shape == [1, 5] + assert packed.shape == (1, 5) assert np.all(packed[0, :] == raw_state['tracer1']) + def test_packs_two_tracers(self): + np.random.seed(0) + dims = ['tracer', '*'] + register_tracer('tracer1', 'g/m^3') + register_tracer('tracer2', 'kg') + raw_state = {'tracer1': np.random.randn(5), 'tracer2': np.random.randn(5)} + packer = TracerPacker(self.component, dims) + packed = packer.pack(raw_state) + assert isinstance(packed, np.ndarray) + assert packed.shape == (2, 5) + + def test_packs_three_tracers_in_order_registered(self): + np.random.seed(0) + dims = ['tracer', '*'] + register_tracer('tracer1', 'g/m^3') + register_tracer('tracer2', 'kg'), + register_tracer('tracer3', 'kg/m^3') + raw_state = { + 'tracer1': np.random.randn(5), + 'tracer2': np.random.randn(5), + 'tracer3': np.random.randn(5), + } + packer = TracerPacker(self.component, dims) + packed = packer.pack(raw_state) + assert isinstance(packed, np.ndarray) + assert packed.shape == (3, 5) + assert np.all(packed[0, :] == raw_state['tracer1']) + assert np.all(packed[1, :] == raw_state['tracer2']) + assert np.all(packed[2, :] == raw_state['tracer3']) + + def test_unpacks_three_tracers_in_order_registered(self): + np.random.seed(0) + dims = ['tracer', '*'] + register_tracer('tracer1', 'g/m^3') + register_tracer('tracer2', 'kg'), + register_tracer('tracer3', 'kg/m^3') + raw_state = { + 'tracer1': np.random.randn(5), + 'tracer2': np.random.randn(5), + 'tracer3': np.random.randn(5), + } + packer = TracerPacker(self.component, dims) + packed = packer.pack(raw_state) + unpacked = packer.unpack(packed) + assert isinstance(unpacked, dict) + assert len(unpacked) == 3 + assert np.all(unpacked['tracer1'] == raw_state['tracer1']) + assert np.all(unpacked['tracer2'] == raw_state['tracer2']) + assert np.all(unpacked['tracer3'] == raw_state['tracer3']) + + +class PrognosticTracerPackerTests(TracerPackerBase, unittest.TestCase): + + def setUp(self): + self.component = MockPrognostic() + super(PrognosticTracerPackerTests, self).setUp() + + def tearDown(self): + self.component = None + super(PrognosticTracerPackerTests, self).tearDown() + + def test_packs_updates_tendency_properties(self): + np.random.seed(0) + dims = ['tracer', '*'] + register_tracer('tracer1', 'g/m^3') + packer = TracerPacker(self.component, dims) + assert 'tracer1' in self.component.tendency_properties + assert tuple(self.component.tendency_properties['tracer1']['dims']) == ('*',) + assert units_are_compatible(self.component.tendency_properties['tracer1']['units'], 'g/m^3 s^-1') + assert len(self.component.tendency_properties) == 1 + + def test_packs_updates_tendency_properties_after_init(self): + np.random.seed(0) + dims = ['tracer', '*'] + packer = TracerPacker(self.component, dims) + assert len(self.component.tendency_properties) == 0 + register_tracer('tracer1', 'g/m^3') + assert 'tracer1' in self.component.tendency_properties + assert tuple( + self.component.tendency_properties['tracer1']['dims']) == ('*',) + assert units_are_compatible(self.component.tendency_properties['tracer1']['units'], 'g/m^3 s^-1') + assert len(self.component.tendency_properties) == 1 + + +class ImplicitPrognosticTracerPackerTests(PrognosticTracerPackerTests): + + def setUp(self): + self.component = MockImplicitPrognostic() + super(ImplicitPrognosticTracerPackerTests, self).setUp() + + def tearDown(self): + self.component = None + super(ImplicitPrognosticTracerPackerTests, self).tearDown() + + +class ImplicitTracerPackerTests(TracerPackerBase, unittest.TestCase): + + def setUp(self): + self.component = MockImplicit() + super(ImplicitTracerPackerTests, self).setUp() + + def tearDown(self): + self.component = None + super(ImplicitTracerPackerTests, self).tearDown() + + def test_packs_updates_output_properties(self): + np.random.seed(0) + dims = ['tracer', '*'] + register_tracer('tracer1', 'g/m^3') + packer = TracerPacker(self.component, dims) + assert 'tracer1' in self.component.output_properties + assert tuple(self.component.output_properties['tracer1']['dims']) == ('*',) + assert self.component.output_properties['tracer1']['units'] == 'g/m^3' + assert len(self.component.output_properties) == 1 + + def test_packs_updates_output_properties_after_init(self): + np.random.seed(0) + dims = ['tracer', '*'] + packer = TracerPacker(self.component, dims) + assert len(self.component.output_properties) == 0 + register_tracer('tracer1', 'g/m^3') + assert 'tracer1' in self.component.output_properties + assert tuple( + self.component.output_properties['tracer1']['dims']) == ('*',) + assert self.component.output_properties['tracer1']['units'] == 'g/m^3' + assert len(self.component.output_properties) == 1 + + +class DiagnosticTracerPackerTests(unittest.TestCase): + + def test_raises_on_diagnostic_init(self): + diagnostic = MockDiagnostic() + with self.assertRaises(TypeError): + TracerPacker(diagnostic, ['tracer', '*']) + + +class TracerComponentBase(object): + + def setUp(self): + clear_tracers() + clear_packers() + + def tearDown(self): + clear_tracers() + clear_packers() + + def call_component(self, input_state): + pass + + def test_packs_no_tracers(self): + input_state = { + 'time': timedelta(0), + } + self.call_component(input_state) + packed = self.component.state_given['tracers'] + assert isinstance(packed, np.ndarray) + assert packed.shape == (0, 0) + + def test_unpacks_no_tracers(self): + input_state = { + 'time': timedelta(0), + } + unpacked = self.call_component(input_state) + assert isinstance(unpacked, dict) + assert len(unpacked) == 0 + + def test_packs_one_tracer(self): + np.random.seed(0) + register_tracer('tracer1', 'g/m^3') + input_state = { + 'time': timedelta(0), + 'tracer1': DataArray( + np.random.randn(5), + dims=['dim1'], + attrs={'units': 'g/m^3'}, + ) + } + self.call_component(input_state) + packed = self.component.state_given['tracers'] + assert isinstance(packed, np.ndarray) + assert packed.shape == (1, 5) + assert np.all(packed[0, :] == input_state['tracer1'].values) + + def test_packs_one_3d_tracer(self): + np.random.seed(0) + register_tracer('tracer1', 'g/m^3') + input_state = { + 'time': timedelta(0), + 'tracer1': DataArray( + np.random.randn(1, 2, 3), + dims=['dim1', 'dim2', 'dim3'], + attrs={'units': 'g/m^3'}, + ) + } + self.call_component(input_state) + packed = self.component.state_given['tracers'] + assert isinstance(packed, np.ndarray) + assert packed.shape == (1, 6) + assert np.all(packed[0, :] == input_state['tracer1'].values.flatten()) + + def test_restores_one_3d_tracer(self): + np.random.seed(0) + register_tracer('tracer1', 'g/m^3') + input_state = { + 'time': timedelta(0), + 'tracer1': DataArray( + np.random.randn(1, 2, 3), + dims=['dim1', 'dim2', 'dim3'], + attrs={'units': 'g/m^3'}, + ) + } + unpacked = self.call_component(input_state) + assert len(unpacked) == 1 + assert unpacked['tracer1'].shape == input_state['tracer1'].shape + assert np.all(unpacked['tracer1'].values == input_state['tracer1'].values) + + def test_updates_input_properties(self): + np.random.seed(0) + register_tracer('tracer1', 'g/m^3') + assert 'tracer1' in self.component.input_properties + assert tuple(self.component.input_properties['tracer1']['dims']) == ('*',) + assert self.component.input_properties['tracer1']['units'] == 'g/m^3' + assert len(self.component.input_properties) == 1 + + def test_packs_two_tracers(self): + np.random.seed(0) + register_tracer('tracer1', 'g/m^3') + register_tracer('tracer2', 'kg') + input_state = { + 'time': timedelta(0), + 'tracer1': DataArray( + np.random.randn(5), + dims=['dim1'], + attrs={'units': 'g/m^3'}, + ), + 'tracer2': DataArray( + np.random.randn(5), + dims=['dim1'], + attrs={'units': 'kg'} + ), + } + unpacked = self.call_component(input_state) + packed = self.component.state_given['tracers'] + assert isinstance(packed, np.ndarray) + assert packed.shape == (2, 5) + assert np.all(packed[0, :] == input_state['tracer1'].values) + assert np.all(packed[1, :] == input_state['tracer2'].values) + assert len(unpacked) == 2 + assert np.all(unpacked['tracer1'].values == input_state['tracer1'].values) + assert np.all(unpacked['tracer2'].values == input_state['tracer2'].values) + + def test_packing_differing_dims(self): + np.random.seed(0) + register_tracer('tracer1', 'g/m^3') + register_tracer('tracer2', 'kg') + input_state = { + 'time': timedelta(0), + 'tracer1': DataArray( + np.random.randn(2), + dims=['dim1'], + attrs={'units': 'g/m^3'}, + ), + 'tracer2': DataArray( + np.random.randn(2), + dims=['dim2'], + attrs={'units': 'kg'} + ), + } + unpacked = self.call_component(input_state) + packed = self.component.state_given['tracers'] + assert isinstance(packed, np.ndarray) + assert packed.shape == (2, 4) + assert len(unpacked) == 2 + + +class PrognosticTracerComponentTests(TracerComponentBase, unittest.TestCase): + + def setUp(self): + super(PrognosticTracerComponentTests, self).setUp() + self.component = MockTracerPrognostic() + + def tearDown(self): + super(PrognosticTracerComponentTests, self).tearDown() + self.component = None + + def call_component(self, input_state): + return self.component(input_state)[0] + + +class ImplicitPrognosticTracerComponentTests(TracerComponentBase, unittest.TestCase): + + def setUp(self): + super(ImplicitPrognosticTracerComponentTests, self).setUp() + self.component = MockTracerImplicitPrognostic() + + def tearDown(self): + super(ImplicitPrognosticTracerComponentTests, self).tearDown() + self.component = None + + def call_component(self, input_state): + return self.component(input_state, timedelta(hours=1))[0] + + +class ImplicitTracerComponentTests(TracerComponentBase, unittest.TestCase): + + def setUp(self): + super(ImplicitTracerComponentTests, self).setUp() + self.component = MockTracerImplicit() + + def tearDown(self): + super(ImplicitTracerComponentTests, self).tearDown() + self.component = None + + def call_component(self, input_state): + return self.component(input_state, timedelta(hours=1))[1] From 7318628f6b9116c308133539b66984fd55c459d3 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 18 Jun 2018 16:10:17 -0700 Subject: [PATCH 78/98] Added documentation for tracers and unit helper functions --- docs/computation.rst | 9 +++++++ docs/index.rst | 2 ++ docs/quickstart.rst | 14 +++++----- docs/tracers.rst | 24 +++++++++++++++++ docs/units.rst | 10 +++++++ docs/writing_components.rst | 52 ++++++++++++++++++++++++++++++++++++- 6 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 docs/tracers.rst create mode 100644 docs/units.rst diff --git a/docs/computation.rst b/docs/computation.rst index 51822b1..b6712ea 100644 --- a/docs/computation.rst +++ b/docs/computation.rst @@ -325,4 +325,13 @@ always excited about new wrapped components. :special-members: :exclude-members: __weakref__,__metaclass__ +Tracer Properties +----------------- + +You may notice some components have properties that mention tracers. +`uses_tracers` is a boolean that tells you whether the component makes use of +tracers or not, while `tracer_dims` contains the dimensions of tracer arrays +used internally by the object. For more on tracers as a user, see :ref:`Tracers`, +and for more on tracers as a component author, see :ref:`Writing Components`. + .. _Python documentation for dicts: https://docs.python.org/3/tutorial/datastructures.html#dictionaries diff --git a/docs/index.rst b/docs/index.rst index afcb1b3..291f215 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,6 +28,8 @@ Documentation computation monitors composites + tracers + units writing_components memory_management contributing diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 169b739..6d1f8f2 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -109,17 +109,17 @@ for us: state = get_initial_state(nx=256, ny=128, nz=64) state['time'] = datetime(2000, 1, 1) -An initialized `state` is a dictionary whose keys are strings (like +An initialized ``state`` is a dictionary whose keys are strings (like 'air_temperature') and values are :py:class:`~sympl.DataArray` objects, which store not only the data but also metadata like units. The one exception -is the "time" quantity which is either a `datetime`-like or `timedelta`-like +is the "time" quantity which is either a ``datetime``-like or ``timedelta``-like object. Here we are calling :py:func:`sympl.datetime` to initialize time, rather than directly creating a Python datetime. This is because :py:func:`sympl.datetime` can support a number of calendars using the -`netcdftime` package, if installed, unlike the built-in `datetime` which only +`netcdftime` package, if installed, unlike the built-in ``datetime`` which only supports the Proleptic Gregorian calendar. -You can read more about the `state`, including :py:func:`sympl.datetime` in +You can read more about the ``state``, including :py:func:`sympl.datetime` in :ref:`Model State`. Initialize Components @@ -139,14 +139,14 @@ Those are the "components": :py:class:`~sympl.AdamsBashforth` is a :py:class:`~sympl.TimeStepper`, which is created with a set of :py:class:`~sympl.Prognostic` components. -The :py:class:`~sympl.Prognostic` components we have here are `Radiation`, -`BoundaryLayer`, and `DeepConvection`. Each of these carries information about +The :py:class:`~sympl.Prognostic` components we have here are ``Radiation``, +``BoundaryLayer``, and ``DeepConvection``. Each of these carries information about what it takes as inputs and provides as outputs, and can be called with a model state to return tendencies for a set of quantities. The :py:class:`~sympl.TimeStepper` uses this information to step the model state forward in time. -The :py:class:`~sympl.UpdateFrequencyWrapper` applied to the `Radiation` object +The :py:class:`~sympl.UpdateFrequencyWrapper` applied to the ``Radiation`` object is an object that acts like a :py:class:`~sympl.Prognostic` but only computes its output if at least a certain amount of model time has passed since the last time the output was computed. Otherwise, it returns the last computed output. diff --git a/docs/tracers.rst b/docs/tracers.rst new file mode 100644 index 0000000..e8cd697 --- /dev/null +++ b/docs/tracers.rst @@ -0,0 +1,24 @@ +======= +Tracers +======= + +In an Earth system model, "tracer" refers to quantities that are passively +moved around in a model without actively interacting with a component. Generally +these are moved around by a dynamical core or subgrid advection scheme. It is +possible for components to do something else to tracers (let us know if you +think of something!) but for now let's assume that's what's going on. + +If a component moves around tracers, it will have its ``uses_tracers`` property +set to ``True``, and will also have a ``tracer_dims`` property set. + +You can tell Sympl that you want components to move a tracer around by +registering it with :py:func:`~sympl.register_tracer`. + +.. autofunction:: sympl.register_tracer + +To see the current list of registered tracers, you can call +:py:func:`~sympl.get_tracer_names` or +:py:func:`~sympl.get_tracer_unit_dict`. + +.. autofunction:: sympl.get_tracer_names +.. autofunction:: sympl.get_tracer_unit_dict diff --git a/docs/units.rst b/docs/units.rst new file mode 100644 index 0000000..5c18921 --- /dev/null +++ b/docs/units.rst @@ -0,0 +1,10 @@ +===== +Units +===== + +While nearly all unit functionality is handled internally within Sympl, it does +expose a few helper functions which may be useful to you. + +.. autofunction:: sympl.is_valid_unit +.. autofunction:: sympl.units_are_same +.. autofunction:: sympl.units_are_compatible diff --git a/docs/writing_components.rst b/docs/writing_components.rst index 7de3de9..ddac639 100644 --- a/docs/writing_components.rst +++ b/docs/writing_components.rst @@ -189,7 +189,7 @@ The Computation *************** That brings us to the ``array_call`` method. In Sympl components, this is the -method which takes in a state dictionary as numpy arrays (*not* ``DataArray``s) +method which takes in a state dictionary as numpy arrays (*not* ``DataArray``) and returns dictionaries with numpy array outputs. .. code-block:: python @@ -283,3 +283,53 @@ Also notice that even though the alias is set in input_properties, it is also used when restoring DataArrays. If there is an output that is not also an input, the alias could instead be set in ``diagnostic_properties``, ``tendency_properties``, or ``output_properties``, wherever is relevant. + + +Using Tracers +------------- + +.. note:: This feature is mostly used in dynamical cores. If you don't think you need + this, you probably don't. + +Sympl's base components have some features to automatically create tracer arrays +for use by dynamical components. If an :py:class:`~sympl.Implicit`, +:py:class:`~sympl.Prognostic`, or :py:class:`~sympl.ImplicitPrognostic` +component specifies ``uses_tracers = True`` and sets ``tracer_dims``, this +feature is enabled. + +.. code-block:: python + + class MyDynamicalCore(Implicit): + + uses_tracers = True + tracer_dims = ['tracer', '*', 'mid_levels'] + + [...] + +``tracer_dims`` is a list or tuple in the form of a ``dims`` attribute on one of its +inputs, and must have a "tracer" dimension. This dimension refers to which +tracer (you could call it "tracer number"). + +Once this feature is enabled, the ``state`` passed to ``array_call`` on the +component will include a quantity called "tracers" with the dimensions +specified by ``tracer_dims``. It will also be required that these tracers +are used in the output. For a :py:class:`~sympl.Implicit` component, "tracers" +must be present in the output state, and for a :py:class:`~sympl.Prognostic` or +:py:class:`~sympl.ImplicitPrognostic` component "tracers" must be present in +the tendencies, with the same dimensions as the input "tracers". + +On these latter two components, you should also specify a +``tracer_tendency_time_unit`` property, which refers to the time part of the +tendency unit. For example, if the input tracer is in units of ``g m^-3``, and +``tracer_tendency_time_unit`` is "s", then the output tendency will be in +units of ``g m^-3 s^-1``. This value is set as "s" (or seconds) by default. + +.. code-block:: python + + class MyDynamicalCore(Prognostic): + + uses_tracers = True + tracer_dims = ['tracer', '*', 'mid_levels'] + tracer_tendency_time_unit = 's' + + [...] From 37ba1594dc44a87f40c1821ed136518497236a80 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 18 Jun 2018 16:18:44 -0700 Subject: [PATCH 79/98] flake8 --- sympl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sympl/__init__.py b/sympl/__init__.py index 334647a..7e7541c 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -47,7 +47,7 @@ get_constants_string, TimeDifferencingWrapper, ensure_no_shared_keys, get_numpy_array, jit, - register_tracer, get_tracer_unit_dict, + register_tracer, get_tracer_unit_dict, get_tracer_names, restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, initialize_numpy_arrays_with_properties, From 4f57a8cebea7ef3aee99733a8689b26d5e71a00f Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 11 Jul 2018 17:34:07 -0700 Subject: [PATCH 80/98] Fixed alias bug in initialize numpy arrays, made _check_self_is_initialized less private --- sympl/_core/base_components.py | 16 +++++++-------- sympl/_core/state.py | 26 ++++++++++++++++++++---- sympl/_core/tracers.py | 36 ++++++++++++++++++++++++++++------ tests/test_state.py | 26 ++++++++++++++++++++++++ tests/test_time.py | 2 +- 5 files changed, 87 insertions(+), 19 deletions(-) diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index c02c40f..9511c19 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -561,7 +561,7 @@ def _get_tendency_name(self, name): def tendencies_in_diagnostics(self): return self._tendencies_in_diagnostics # value cannot be modified - def __check_self_is_initialized(self): + def _check_self_is_initialized(self): try: initialized = self.__initialized except AttributeError: @@ -605,7 +605,7 @@ def __call__(self, state, timestep): If state is not a valid input for the Implicit instance for other reasons. """ - self.__check_self_is_initialized() + self._check_self_is_initialized() self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) if self.uses_tracers: @@ -786,7 +786,7 @@ def _insert_tendency_properties(self): def _get_tendency_name(self, name): return '{}_tendency_from_{}'.format(name, self.name) - def __check_self_is_initialized(self): + def _check_self_is_initialized(self): try: initialized = self.__initialized except AttributeError: @@ -829,7 +829,7 @@ def __call__(self, state): InvalidStateError If state is not a valid input for the Prognostic instance. """ - self.__check_self_is_initialized() + self._check_self_is_initialized() self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) if self.uses_tracers: @@ -1005,7 +1005,7 @@ def _insert_tendency_properties(self): def _get_tendency_name(self, name): return '{}_tendency_from_{}'.format(name, self.name) - def __check_self_is_initialized(self): + def _check_self_is_initialized(self): try: initialized = self.__initialized except AttributeError: @@ -1050,7 +1050,7 @@ def __call__(self, state, timestep): InvalidStateError If state is not a valid input for the Prognostic instance. """ - self.__check_self_is_initialized() + self._check_self_is_initialized() self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) if self.uses_tracers: @@ -1163,7 +1163,7 @@ def __init__(self): self.__initialized = True super(Diagnostic, self).__init__() - def __check_self_is_initialized(self): + def _check_self_is_initialized(self): try: initialized = self.__initialized except AttributeError: @@ -1201,7 +1201,7 @@ def __call__(self, state): InvalidStateError If state is not a valid input for the Prognostic instance. """ - self.__check_self_is_initialized() + self._check_self_is_initialized() self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) raw_state['time'] = state['time'] diff --git a/sympl/_core/state.py b/sympl/_core/state.py index 6d6b168..1bd0d68 100644 --- a/sympl/_core/state.py +++ b/sympl/_core/state.py @@ -1,6 +1,7 @@ from .exceptions import InvalidStateError, InvalidPropertyDictError import numpy as np from .array import DataArray +from .tracers import get_tracer_names def copy_untouched_quantities(old_state, new_state): @@ -146,7 +147,7 @@ def get_numpy_array(data_array, out_dims, dim_lengths): def initialize_numpy_arrays_with_properties( - output_properties, raw_input_state, input_properties): + output_properties, raw_input_state, input_properties, tracer_dims=None): """ Parameters ---------- @@ -181,18 +182,35 @@ def initialize_numpy_arrays_with_properties( dims_from_out_properties = extract_output_dims_properties( output_properties, input_properties, []) out_dict = {} + tracer_names = get_tracer_names() for name, out_dims in dims_from_out_properties.items(): + if tracer_dims is None or name not in tracer_names: + out_shape = [] + for dim in out_dims: + out_shape.append(dim_lengths[dim]) + dtype = output_properties[name].get('dtype', np.float64) + out_dict[name] = np.zeros(out_shape, dtype=dtype) + if tracer_dims is not None: out_shape = [] - for dim in out_dims: + dim_lengths['tracer'] = len(tracer_names) + for dim in tracer_dims: out_shape.append(dim_lengths[dim]) - dtype = output_properties[name].get('dtype', np.float64) - out_dict[name] = np.zeros(out_shape, dtype=dtype) + out_dict['tracers'] = np.zeros(out_shape, dtype=np.float64) return out_dict +def properties_include_tracers(input_properties): + for properties in input_properties.values(): + if properties.get('tracer', False): + return True + return False + + def get_dim_lengths_from_raw_input(raw_input, input_properties): dim_lengths = {} for name, properties in input_properties.items(): + if 'alias' in properties.keys(): + name = properties['alias'] for i, dim_name in enumerate(properties['dims']): if dim_name in dim_lengths: if raw_input[name].shape[i] != dim_lengths[dim_name]: diff --git a/sympl/_core/tracers.py b/sympl/_core/tracers.py index 3666db6..c7056c7 100644 --- a/sympl/_core/tracers.py +++ b/sympl/_core/tracers.py @@ -75,7 +75,8 @@ def get_quantity_dims(tracer_dims): class TracerPacker(object): - def __init__(self, component, tracer_dims): + def __init__(self, component, tracer_dims, prepend_tracers=None): + self._prepend_tracers = prepend_tracers or () self._tracer_dims = tuple(tracer_dims) self._tracer_quantity_dims = get_quantity_dims(tracer_dims) if hasattr(component, 'tendency_properties') or hasattr(component, 'output_properties'): @@ -85,6 +86,9 @@ def __init__(self, component, tracer_dims): 'Expected a component object subclassing type Implicit, ' 'ImplicitPrognostic, or Prognostic but received component of ' 'type {}'.format(component.__class__.__name__)) + for name, units in self._prepend_tracers: + if name not in _tracer_unit_dict.keys(): + self.insert_tracer_to_properties(name, units) for name, units in _tracer_unit_dict.items(): self.insert_tracer_to_properties(name, units) _packers.add(self) @@ -97,6 +101,13 @@ def insert_tracer_to_properties(self, name, units): self._insert_tracer_to_output_properties(name, units) def _insert_tracer_to_input_properties(self, name, units): + if name in self.component.input_properties.keys(): + raise InvalidPropertyDictError( + 'Attempted to insert {} as tracer to component of type {} but ' + 'it already has that quantity defined as an input.'.format( + name, self.component.__class__.__name__ + ) + ) if name not in self.component.input_properties: self.component.input_properties[name] = { 'dims': self._tracer_quantity_dims, @@ -105,6 +116,13 @@ def _insert_tracer_to_input_properties(self, name, units): } def _insert_tracer_to_output_properties(self, name, units): + if name in self.component.output_properties.keys(): + raise InvalidPropertyDictError( + 'Attempted to insert {} as tracer to component of type {} but ' + 'it already has that quantity defined as an output.'.format( + name, self.component.__class__.__name__ + ) + ) if name not in self.component.output_properties: self.component.output_properties[name] = { 'dims': self._tracer_quantity_dims, @@ -114,6 +132,14 @@ def _insert_tracer_to_output_properties(self, name, units): def _insert_tracer_to_tendency_properties(self, name, units): time_unit = getattr(self.component, 'tracer_tendency_time_unit', 's') + if name in self.component.tendency_properties.keys(): + raise InvalidPropertyDictError( + 'Attempted to insert {} as tracer to component of type {} but ' + 'it already has that quantity defined as a tendency ' + 'output.'.format( + name, self.component.__class__.__name__ + ) + ) if name not in self.component.tendency_properties: self.component.tendency_properties[name] = { 'dims': self._tracer_quantity_dims, @@ -121,10 +147,6 @@ def _insert_tracer_to_tendency_properties(self, name, units): 'tracer': True, } - def remove_tracer_from_properties(self, name): - if self.is_tracer(name): - self.component.pop(name) - def is_tracer(self, tracer_name): return self.component.input_properties.get(tracer_name, {}).get('tracer', False) @@ -147,8 +169,10 @@ def register_tracers(self, unit_dict, input_properties): @property def tracer_names(self): return_list = [] + for name, units in self._prepend_tracers: + return_list.append(name) for name in _tracer_names: - if self.is_tracer(name): + if name not in return_list and self.is_tracer(name): return_list.append(name) return tuple(return_list) diff --git a/tests/test_state.py b/tests/test_state.py index aaa6040..62bd162 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -41,6 +41,32 @@ def test_single_output_single_dim(self): assert np.all(result['output1'] == np.zeros([10])) assert result['output1'].dtype == np.float64 + def test_single_output_single_dim_aliased(self): + output_properties = { + 'output1': { + 'dims': ['dim1'], + 'units': 'm', + } + } + input_properties = { + 'input1': { + 'dims': ['dim1'], + 'units': 's^-1', + 'alias': 'in1' + } + } + input_state = { + 'in1': np.zeros([10]) + } + + result = initialize_numpy_arrays_with_properties( + output_properties, input_state, input_properties) + assert len(result.keys()) == 1 + assert 'output1' in result.keys() + assert result['output1'].shape == (10,) + assert np.all(result['output1'] == np.zeros([10])) + assert result['output1'].dtype == np.float64 + def test_single_output_single_dim_custom_dtype(self): output_properties = { 'output1': { diff --git a/tests/test_time.py b/tests/test_time.py index c6a9712..e14462c 100644 --- a/tests/test_time.py +++ b/tests/test_time.py @@ -174,7 +174,7 @@ def dt_class(self): return ct.DatetimeJulian -unittest.skipIf(ct is None, netcdftime_not_installed) +@unittest.skipIf(ct is None, netcdftime_not_installed) class GregorianTests(unittest.TestCase, DatetimeBase): calendar = 'gregorian' From cb1b42d6852bf8970e3663853002569ee27df8ee Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 12 Jul 2018 12:26:20 -0700 Subject: [PATCH 81/98] Fixed bug that caused prepend_tracers not to work --- sympl/_core/base_components.py | 12 +++- sympl/_core/state.py | 8 ++- tests/test_tracers.py | 126 +++++++++++++++++++++++++++++---- 3 files changed, 127 insertions(+), 19 deletions(-) diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 9511c19..5a03cbd 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -509,7 +509,9 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): raise ValueError( 'Component of type {} must specify tracer_dims property ' 'when uses_tracers=True'.format(self.__class__.__name__)) - self._tracer_packer = TracerPacker(self, self.tracer_dims) + prepend_tracers = getattr(self, 'prepend_tracers', None) + self._tracer_packer = TracerPacker( + self, self.tracer_dims, prepend_tracers=prepend_tracers) super(Implicit, self).__init__() def _insert_tendency_properties(self): @@ -760,7 +762,9 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): raise ValueError( 'Component of type {} must specify tracer_dims property ' 'when uses_tracers=True'.format(self.__class__.__name__)) - self._tracer_packer = TracerPacker(self, self.tracer_dims) + prepend_tracers = getattr(self, 'prepend_tracers', None) + self._tracer_packer = TracerPacker( + self, self.tracer_dims, prepend_tracers=prepend_tracers) self.__initialized = True super(Prognostic, self).__init__() @@ -979,7 +983,9 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): raise ValueError( 'Component of type {} must specify tracer_dims property ' 'when uses_tracers=True'.format(self.__class__.__name__)) - self._tracer_packer = TracerPacker(self, self.tracer_dims) + prepend_tracers = getattr(self, 'prepend_tracers', None) + self._tracer_packer = TracerPacker( + self, self.tracer_dims, prepend_tracers=prepend_tracers) self.__initialized = True super(ImplicitPrognostic, self).__init__() diff --git a/sympl/_core/state.py b/sympl/_core/state.py index 1bd0d68..97ba843 100644 --- a/sympl/_core/state.py +++ b/sympl/_core/state.py @@ -147,7 +147,8 @@ def get_numpy_array(data_array, out_dims, dim_lengths): def initialize_numpy_arrays_with_properties( - output_properties, raw_input_state, input_properties, tracer_dims=None): + output_properties, raw_input_state, input_properties, tracer_dims=None, + prepend_tracers=()): """ Parameters ---------- @@ -182,7 +183,8 @@ def initialize_numpy_arrays_with_properties( dims_from_out_properties = extract_output_dims_properties( output_properties, input_properties, []) out_dict = {} - tracer_names = get_tracer_names() + tracer_names = list(get_tracer_names()) + tracer_names.extend(entry[0] for entry in prepend_tracers) for name, out_dims in dims_from_out_properties.items(): if tracer_dims is None or name not in tracer_names: out_shape = [] @@ -209,6 +211,8 @@ def properties_include_tracers(input_properties): def get_dim_lengths_from_raw_input(raw_input, input_properties): dim_lengths = {} for name, properties in input_properties.items(): + if properties.get('tracer', False): + continue if 'alias' in properties.keys(): name = properties['alias'] for i, dim_name in enumerate(properties['dims']): diff --git a/tests/test_tracers.py b/tests/test_tracers.py index 5ff4c6f..4e8e08f 100644 --- a/tests/test_tracers.py +++ b/tests/test_tracers.py @@ -41,6 +41,9 @@ class MockTracerPrognostic(Prognostic): tracer_dims = ('tracer', '*') def __init__(self, **kwargs): + prepend_tracers = kwargs.pop('prepend_tracers', None) + if prepend_tracers is not None: + self.prepend_tracers = prepend_tracers self.input_properties = {} self.diagnostic_properties = {} self.tendency_properties = {} @@ -91,7 +94,10 @@ class MockTracerImplicitPrognostic(ImplicitPrognostic): uses_tracers = True tracer_dims = ('tracer', '*') - def __init__( self, **kwargs): + def __init__(self, **kwargs): + prepend_tracers = kwargs.pop('prepend_tracers', None) + if prepend_tracers is not None: + self.prepend_tracers = prepend_tracers self.input_properties = {} self.diagnostic_properties = {} self.tendency_properties = {} @@ -164,6 +170,9 @@ class MockTracerImplicit(Implicit): tracer_dims = ('tracer', '*') def __init__(self, **kwargs): + prepend_tracers = kwargs.pop('prepend_tracers', None) + if prepend_tracers is not None: + self.prepend_tracers = prepend_tracers self.input_properties = {} self.diagnostic_properties = {} self.output_properties = {} @@ -183,19 +192,6 @@ def array_call(self, state, timestep): return_state.pop('time') return self.diagnostic_output, return_state -""" -On init and tracer registration, should update input properties of its -component. - -On pack, should pack all tracers not already present in input_properties. - -On unpack, should unpack all tracers not present in input_properties - -Keep track of internal representation of tracer order. - On adding new tracer, add it to the end. - Allow global tracer order to be set. -""" - class RegisterTracerTests(unittest.TestCase): @@ -508,6 +504,108 @@ def test_packs_one_tracer(self): assert packed.shape == (1, 5) assert np.all(packed[0, :] == input_state['tracer1'].values) + def test_packs_one_prepended_tracer(self): + np.random.seed(0) + self.component = self.component.__class__(prepend_tracers=[('tracer1', 'g/m^3')]) + input_state = { + 'time': timedelta(0), + 'tracer1': DataArray( + np.random.randn(5), + dims=['dim1'], + attrs={'units': 'g/m^3'}, + ) + } + self.call_component(input_state) + packed = self.component.state_given['tracers'] + assert isinstance(packed, np.ndarray) + assert packed.shape == (1, 5) + assert np.all(packed[0, :] == input_state['tracer1'].values) + + def test_packs_two_prepended_tracers(self): + np.random.seed(0) + self.component = self.component.__class__( + prepend_tracers=[('tracer1', 'g/m^3'), ('tracer2', 'J/m^3')]) + input_state = { + 'time': timedelta(0), + 'tracer1': DataArray( + np.random.randn(5), + dims=['dim1'], + attrs={'units': 'g/m^3'}, + ), + 'tracer2': DataArray( + np.random.randn(5), + dims=['dim1'], + attrs={'units': 'J/m^3'}, + ), + } + unpacked = self.call_component(input_state) + packed = self.component.state_given['tracers'] + assert isinstance(packed, np.ndarray) + assert packed.shape == (2, 5) + assert np.all(packed[0, :] == input_state['tracer1'].values) + assert np.all(packed[1, :] == input_state['tracer2'].values) + assert len(unpacked) == 2 + assert np.all(unpacked['tracer1'].values == input_state['tracer1'].values) + assert np.all(unpacked['tracer2'].values == input_state['tracer2'].values) + + def test_packs_prepended_and_normal_tracers_register_first(self): + register_tracer('tracer2', 'J/m^3') + self.component = self.component.__class__( + prepend_tracers=[('tracer1', 'g/m^3')]) + input_state = { + 'time': timedelta(0), + 'tracer1': DataArray( + np.random.randn(5), + dims=['dim1'], + attrs={'units': 'g/m^3'}, + ), + 'tracer2': DataArray( + np.random.randn(5), + dims=['dim1'], + attrs={'units': 'J/m^3'}, + ), + } + unpacked = self.call_component(input_state) + packed = self.component.state_given['tracers'] + assert isinstance(packed, np.ndarray) + assert packed.shape == (2, 5) + assert np.all(packed[0, :] == input_state['tracer1'].values) + assert np.all(packed[1, :] == input_state['tracer2'].values) + assert len(unpacked) == 2 + assert np.all( + unpacked['tracer1'].values == input_state['tracer1'].values) + assert np.all( + unpacked['tracer2'].values == input_state['tracer2'].values) + + def test_packs_prepended_and_normal_tracers_register_after_init(self): + self.component = self.component.__class__( + prepend_tracers=[('tracer1', 'g/m^3')]) + register_tracer('tracer2', 'J/m^3') + input_state = { + 'time': timedelta(0), + 'tracer1': DataArray( + np.random.randn(5), + dims=['dim1'], + attrs={'units': 'g/m^3'}, + ), + 'tracer2': DataArray( + np.random.randn(5), + dims=['dim1'], + attrs={'units': 'J/m^3'}, + ), + } + unpacked = self.call_component(input_state) + packed = self.component.state_given['tracers'] + assert isinstance(packed, np.ndarray) + assert packed.shape == (2, 5) + assert np.all(packed[0, :] == input_state['tracer1'].values) + assert np.all(packed[1, :] == input_state['tracer2'].values) + assert len(unpacked) == 2 + assert np.all( + unpacked['tracer1'].values == input_state['tracer1'].values) + assert np.all( + unpacked['tracer2'].values == input_state['tracer2'].values) + def test_packs_one_3d_tracer(self): np.random.seed(0) register_tracer('tracer1', 'g/m^3') From d0b4be6fdf21c9eddf093e267ef992204b30b2c0 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Sat, 21 Jul 2018 09:28:11 -0700 Subject: [PATCH 82/98] Renamed base classes --- HISTORY.rst | 29 ++- docs/composites.rst | 10 +- docs/computation.rst | 84 +++---- docs/memory_management.rst | 6 +- docs/overview.rst | 18 +- docs/quickstart.rst | 20 +- docs/state.rst | 2 +- docs/timestepping.rst | 34 +-- docs/writing_components.rst | 30 +-- sympl/__init__.py | 16 +- sympl/_components/__init__.py | 4 +- sympl/_components/basic.py | 34 +-- sympl/_components/netcdf.py | 2 +- sympl/_components/timesteppers.py | 22 +- sympl/_core/base_components.py | 62 ++--- sympl/_core/composite.py | 34 +-- .../{timestepper.py => prognosticstepper.py} | 56 ++--- sympl/_core/state.py | 9 +- sympl/_core/tracers.py | 4 +- sympl/_core/util.py | 4 +- sympl/_core/wrappers.py | 30 +-- tests/test_base_components.py | 178 +++++++------- tests/test_components.py | 28 +-- tests/test_composite.py | 232 +++++++++--------- tests/test_time_differencing_wrapper.py | 16 +- tests/test_timestepping.py | 102 ++++---- tests/test_tracers.py | 44 ++-- tests/test_util.py | 6 +- tests/test_wrapper.py | 52 ++-- 29 files changed, 586 insertions(+), 582 deletions(-) rename sympl/_core/{timestepper.py => prognosticstepper.py} (82%) diff --git a/HISTORY.rst b/HISTORY.rst index d30a79c..ce2ee73 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,15 +5,15 @@ What's New Latest ------ -* Implicit, Diagnostic, ImplicitPrognostic, and Prognostic base classes were +* Stepper, DiagnosticComponent, ImplicitPrognosticComponent, and PrognosticComponent base classes were modified to include functionality that was previously in ScalingWrapper, UpdateFrequencyWrapper, and TendencyInDiagnosticsWrapper. The functionality of - TendencyInDiagnosticsWrapper is now to be used in Implicit and TimeStepper objects. + TendencyInDiagnosticsWrapper is now to be used in Stepper and PrognosticStepper objects. * Composites now have a component_list attribute which contains the components being composited. * TimeSteppers now have a prognostic_list attribute which contains the prognostics used to calculate tendencies. -* TimeSteppers from sympl can now handle ImplicitPrognostic components. +* TimeSteppers from sympl can now handle ImplicitPrognosticComponent components. * Added a check for netcdftime having the required objects, to fall back on not using netcdftime when those are missing. This is because most objects are missing in older versions of netcdftime (that come packaged with netCDF4) (closes #23). @@ -30,8 +30,8 @@ Latest respectively if outputs do not match. * Added a priority order of property types for determining which aliases are returned by get_component_aliases. -* Fixed a bug where TimeStepper objects would modify the arrays passed to them by - Prognostic objects, leading to unexpected value changes. +* Fixed a bug where PrognosticStepper objects would modify the arrays passed to them by + PrognosticComponent objects, leading to unexpected value changes. * Fixed a bug where constants were missing from the string returned by get_constants_string, particularly any new constants (issue #27) * Fixed a bug in NetCDFMonitor which led to some aliases being skipped. @@ -54,6 +54,9 @@ Latest Breaking changes ~~~~~~~~~~~~~~~~ +* Implicit, Timestepper, Prognostic, ImplicitPrognostic, and Diagnostic objects have been renamed to + PrognosticStepper, Stepper, PrognosticComponent, ImplicitPrognosticComponent, + and DiagnosticComponent. These changes are also reflected in subclass names. * inputs, outputs, diagnostics, and tendencies are no longer attributes of components. In order to get these, you should use e.g. input_properties.keys() * properties dictionaries are now abstract methods, so subclasses must define them. @@ -71,13 +74,13 @@ Breaking changes have been removed. The functionality of these wrappers has been moved to the component base types as methods and initialization options. * 'time' now must be present in the model state dictionary. This is strictly required - for calls to Diagnostic, Prognostic, ImplicitPrognostic, and Implicit components, + for calls to DiagnosticComponent, PrognosticComponent, ImplicitPrognosticComponent, and Stepper components, and may be strictly required in other ways in the future * Removed everything to do with directional wildcards. Currently '*' is the only wildcard dimension. 'x', 'y', and 'z' refer to their own names only. * Removed the combine_dimensions function, which wasn't used anywhere and no longer has much purpose without directional wildcards -* RelaxationPrognostic no longer allows caching of equilibrium values or +* RelaxationPrognosticComponent no longer allows caching of equilibrium values or timescale. They must be provided through the input state. This is to ensure proper conversion of dimensions and units. * Removed ComponentTestBase from package. All of its tests except for output @@ -90,7 +93,7 @@ Breaking changes used instead. If present, `dims` from input properties will be used as default. * Components will now raise an exception when __call__ of the component base - class (e.g. Implicit, Prognostic, etc.) if the __init__ method of the base + class (e.g. Stepper, PrognosticComponent, etc.) if the __init__ method of the base class has not been called, telling the user that the component __init__ method should make a call to the superclass init. @@ -132,15 +135,15 @@ v0.3.0 restore_data_arrays_with_properties * corrected heat capacity of snow and ice to be floats instead of ints * Added get_constant function as the way to retrieve constants -* Added ImplicitPrognostic as a new component type. It is like a Prognostic, +* Added ImplicitPrognosticComponent as a new component type. It is like a PrognosticComponent, but its call signature also requires that a timestep be given. -* Added TimeDifferencingWrapper, which turns an Implicit into an - ImplicitPrognostic by applying first-order time differencing. +* Added TimeDifferencingWrapper, which turns an Stepper into an + ImplicitPrognosticComponent by applying first-order time differencing. * Added set_condensible_name as a way of changing what condensible aliases (for example, density_of_solid_phase) refer to. Default is 'water'. * Moved wrappers to their own file (out from util.py). -* Corrected str representation of Diagnostic to say Diagnostic instead of - Implicit. +* Corrected str representation of DiagnosticComponent to say DiagnosticComponent instead of + Stepper. * Added a function reset_constants to reset the constants library to its initial state. * Added a function datetime which accepts calendar as a keyword argument, and diff --git a/docs/composites.rst b/docs/composites.rst index 2c24fd1..5723806 100644 --- a/docs/composites.rst +++ b/docs/composites.rst @@ -4,7 +4,7 @@ Composites There are a set of objects in Sympl that wrap multiple components into a single object so they can be called as if they were one component. There is one each -for :py:class:`~sympl.Prognostic`, :py:class:`~sympl.Diagnostic`, and +for :py:class:`~sympl.PrognosticComponent`, :py:class:`~sympl.DiagnosticComponent`, and :py:class:`~sympl.Monitor`. These can be used to simplify code, so that the way you call a list of components is the same as the way you would call a single component. For example, *instead* of writing: @@ -35,7 +35,7 @@ You could write: .. code-block:: python - prognostic_composite = PrognosticComposite([ + prognostic_composite = PrognosticComponentComposite([ MyPrognostic(), MyOtherPrognostic(), YetAnotherPrognostic(), @@ -46,7 +46,7 @@ This second call is much cleaner. It will also automatically detect whether multiple components are trying to write out the same diagnostic, and raise an exception if that is the case (so no results are being silently overwritten). You can get similar simplifications for -:py:class:`~sympl.Diagnostic` and :py:class:`~sympl.Monitor`. +:py:class:`~sympl.DiagnosticComponent` and :py:class:`~sympl.Monitor`. .. note:: PrognosticComposites are mainly useful inside of TimeSteppers, so if you're only writing a model script it's unlikely you'll need them. @@ -54,12 +54,12 @@ overwritten). You can get similar simplifications for API Reference ------------- -.. autoclass:: sympl.PrognosticComposite +.. autoclass:: sympl.PrognosticComponentComposite :members: :special-members: :exclude-members: __weakref__,__metaclass__ -.. autoclass:: sympl.DiagnosticComposite +.. autoclass:: sympl.DiagnosticComponentComposite :members: :special-members: :exclude-members: __weakref__,__metaclass__ diff --git a/docs/computation.rst b/docs/computation.rst index b6712ea..e8f464e 100644 --- a/docs/computation.rst +++ b/docs/computation.rst @@ -2,17 +2,17 @@ Component Types =============== -In Sympl, computation is mainly performed using :py:class:`~sympl.Prognostic`, -:py:class:`~sympl.Diagnostic`, and :py:class:`~sympl.Implicit` objects. +In Sympl, computation is mainly performed using :py:class:`~sympl.PrognosticComponent`, +:py:class:`~sympl.DiagnosticComponent`, and :py:class:`~sympl.Stepper` objects. Each of these types, once initialized, can be passed in a current model state. -:py:class:`~sympl.Prognostic` objects use the state to return tendencies and -diagnostics at the current time. :py:class:`~sympl.Diagnostic` objects -return only diagnostics from the current time. :py:class:`~sympl.Implicit` +:py:class:`~sympl.PrognosticComponent` objects use the state to return tendencies and +diagnostics at the current time. :py:class:`~sympl.DiagnosticComponent` objects +return only diagnostics from the current time. :py:class:`~sympl.Stepper` objects will take in a timestep along with the state, and then return the next state as well as modifying the current state to include more diagnostics -(it is similar to a :py:class:`~sympl.TimeStepper` in how it is called). +(it is similar to a :py:class:`~sympl.PrognosticStepper` in how it is called). -In specific cases, it may be necessary to use a :py:class:`~sympl.ImplicitPrognostic` +In specific cases, it may be necessary to use a :py:class:`~sympl.ImplicitPrognosticComponent` object, which is discussed at the end of this section. These classes themselves (listed in the previous paragraph) are not ones you @@ -27,15 +27,15 @@ for their inputs and outputs, which are described in the section Details on the internals of components and how to write them are in the section on :ref:`Writing Components`. -Prognostic +PrognosticComponent ---------- -As stated above, :py:class:`~sympl.Prognostic` objects use the state to return +As stated above, :py:class:`~sympl.PrognosticComponent` objects use the state to return tendencies and diagnostics at the current time. In a full model, the tendencies -are used by a time stepping scheme (in Sympl, a :py:class:`~sympl.TimeStepper`) +are used by a time stepping scheme (in Sympl, a :py:class:`~sympl.PrognosticStepper`) to determine the values of quantities at the next time. -You can call a :py:class:`~sympl.Prognostic` directly to get diagnostics and +You can call a :py:class:`~sympl.PrognosticComponent` directly to get diagnostics and tendencies like so: .. code-block:: python @@ -44,35 +44,35 @@ tendencies like so: diagnostics, tendencies = radiation(state) ``diagnostics`` and ``tendencies`` in this case will both be dictionaries, -similar to ``state``. Even if the :py:class:`~sympl.Prognostic` being called +similar to ``state``. Even if the :py:class:`~sympl.PrognosticComponent` being called does not compute any diagnostics, it will still return an empty diagnostics dictionary. -Usually, you will call a Prognostic object through a -:py:class:`~sympl.TimeStepper` that uses it to determine values at the next +Usually, you will call a PrognosticComponent object through a +:py:class:`~sympl.PrognosticStepper` that uses it to determine values at the next timestep. -.. autoclass:: sympl.Prognostic +.. autoclass:: sympl.PrognosticComponent :members: :special-members: :exclude-members: __weakref__,__metaclass__ -.. autoclass:: sympl.ConstantPrognostic +.. autoclass:: sympl.ConstantPrognosticComponent :members: :special-members: :exclude-members: __weakref__,__metaclass__ -.. autoclass:: sympl.RelaxationPrognostic +.. autoclass:: sympl.RelaxationPrognosticComponent :members: :special-members: :exclude-members: __weakref__,__metaclass__ -Diagnostic +DiagnosticComponent ---------- -:py:class:`~sympl.Diagnostic` objects use the state to return quantities +:py:class:`~sympl.DiagnosticComponent` objects use the state to return quantities ('diagnostics') from the same timestep as the input state. You can call a -:py:class:`~sympl.Diagnostic` directly to get diagnostic quantities like so: +:py:class:`~sympl.DiagnosticComponent` directly to get diagnostic quantities like so: .. code-block:: python @@ -80,8 +80,8 @@ Diagnostic diagnostics = diagnostic_component(state) You should be careful to check in the documentation of the particular -:py:class:`~sympl.Diagnostic` you are using to see whether it modifies the -``state`` given to it as input. :py:class:`~sympl.Diagnostic` objects in charge +:py:class:`~sympl.DiagnosticComponent` you are using to see whether it modifies the +``state`` given to it as input. :py:class:`~sympl.DiagnosticComponent` objects in charge of updating ghost cells in particular may modify the arrays in the input dictionary, so that the arrays in the returned ``diagnostics`` dictionary are the same ones as were sent as input in the ``state``. To make it clear that @@ -92,22 +92,22 @@ syntax like: state.update(diagnostic_component(state)) -.. autoclass:: sympl.Diagnostic +.. autoclass:: sympl.DiagnosticComponent :members: :special-members: :exclude-members: __weakref__,__metaclass__ -.. autoclass:: sympl.ConstantDiagnostic +.. autoclass:: sympl.ConstantDiagnosticComponent :members: :special-members: :exclude-members: __weakref__,__metaclass__ -Implicit +Stepper -------- -:py:class:`~sympl.Implicit` objects use a state and a timestep to return the next +:py:class:`~sympl.Stepper` objects use a state and a timestep to return the next state, and update the input state with any relevant diagnostic quantities. You -can call an Implicit object like so: +can call an Stepper object like so: .. code-block:: python @@ -127,7 +127,7 @@ current state. Or if an object provides 'cloud_fraction' as a diagnostic, it may modify an existing 'cloud_fraction' array in the input state if one is present, instead of allocating a new array. -.. autoclass:: sympl.Implicit +.. autoclass:: sympl.Stepper :members: :special-members: :exclude-members: __weakref__,__metaclass__ @@ -140,7 +140,7 @@ there are a number of attributes with names like ``input_properties`` for components. These attributes give a fairly complete description of the inputs and outputs of the component. -You can access them like this (for an example :py:class:`~sympl.Prognostic` +You can access them like this (for an example :py:class:`~sympl.PrognosticComponent` class ``RRTMRadiation``): .. code-block:: python @@ -282,45 +282,45 @@ equal to: that the object will output ``cloud_fraction`` in its diagnostics on the same grid as ``air_temperature``, in dimensionless units. -ImplicitPrognostic +ImplicitPrognosticComponent ------------------ .. warning:: This component type should be avoided unless you know you need it, for reasons discussed in this section. In addition to the component types described above, computation may be performed by a -:py:class:`~sympl.ImplicitPrognostic`. This class should be avoided unless you +:py:class:`~sympl.ImplicitPrognosticComponent`. This class should be avoided unless you know what you are doing, but it may be necessary in certain cases. An -:py:class:`~sympl.ImplicitPrognostic`, like a :py:class:`~sympl.Prognostic`, +:py:class:`~sympl.ImplicitPrognosticComponent`, like a :py:class:`~sympl.PrognosticComponent`, calculates tendencies, but it does so using both the model state and a timestep. Certain components, like ones handling advection using a spectral method, may -need to derive tendencies from an :py:class:`~sympl.Implicit` object by -representing it using an :py:class:`~sympl.ImplicitPrognostic`. +need to derive tendencies from an :py:class:`~sympl.Stepper` object by +representing it using an :py:class:`~sympl.ImplicitPrognosticComponent`. -The reason to avoid using an :py:class:`~sympl.ImplicitPrognostic` is that if +The reason to avoid using an :py:class:`~sympl.ImplicitPrognosticComponent` is that if a component requires a timestep, it is making internal assumptions about how you are timestepping. For example, it may use the timestep to ensure that all supersaturated water is condensed by the end of the timestep using an assumption -about the timestepping. However, if you use a :py:class:`~sympl.TimeStepper` +about the timestepping. However, if you use a :py:class:`~sympl.PrognosticStepper` which does not obey those assumptions, you may get unintended behavior, such as some supersaturated water remaining, or too much water being condensed. -For this reason, the :py:class:`~sympl.TimeStepper` objects included in Sympl -do not wrap :py:class:`~sympl.ImplicitPrognostic` components. If you would like +For this reason, the :py:class:`~sympl.PrognosticStepper` objects included in Sympl +do not wrap :py:class:`~sympl.ImplicitPrognosticComponent` components. If you would like to use this type of component, and know what you are doing, it is pretty easy -to write your own :py:class:`~sympl.TimeStepper` to do so (you can base the code +to write your own :py:class:`~sympl.PrognosticStepper` to do so (you can base the code off of the code in Sympl), or the model you are using might already have components to do this for you. If you are wrapping a parameterization and notice that it needs a timestep to compute its tendencies, that is likely *not* a good reason to write an -:py:class:`~sympl.ImplicitPrognostic`. If at all possible you should modify the +:py:class:`~sympl.ImplicitPrognosticComponent`. If at all possible you should modify the code to compute the value at the next timestep, and write an -:py:class:`~sympl.Implicit` component. You are welcome to reach out to the +:py:class:`~sympl.Stepper` component. You are welcome to reach out to the developers of Sympl if you would like advice on your specific situation! We're always excited about new wrapped components. -.. autoclass:: sympl.ImplicitPrognostic +.. autoclass:: sympl.ImplicitPrognosticComponent :members: :special-members: :exclude-members: __weakref__,__metaclass__ diff --git a/docs/memory_management.rst b/docs/memory_management.rst index f7164d4..f337276 100644 --- a/docs/memory_management.rst +++ b/docs/memory_management.rst @@ -10,15 +10,15 @@ Arrays If possible, you should try to be aware of when there are two code references to the same in-memory array. This can help avoid some common bugs. Let's start -with an example. Say you create a ConstantPrognostic object like so:: +with an example. Say you create a ConstantPrognosticComponent object like so:: >>> import numpy as np - >>> from sympl import ConstantPrognostic, DataArray + >>> from sympl import ConstantPrognosticComponent, DataArray >>> array = DataArray( np.ones((5, 5, 10)), dims=('lon', 'lat', 'lev'), attrs={'units': 'K/s'}) >>> tendencies = {'air_temperature': array} - >>> prognostic = ConstantPrognostic(tendencies) + >>> prognostic = ConstantPrognosticComponent(tendencies) This is all fine so far. But it's important to know that now ``array`` is the same array stored inside ``prognostic``:: diff --git a/docs/overview.rst b/docs/overview.rst index 4f1b4b5..6cbf3e3 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -73,21 +73,21 @@ set to initial values. Code to do this may be present in other packages, or you can write this code yourself. The state and its initialization is discussed further in :ref:`Model State`. -The state dictionary is evolved by :py:class:`~sympl.TimeStepper` and -:py:class:`~sympl.Implicit` objects. These types of objects take in the state +The state dictionary is evolved by :py:class:`~sympl.PrognosticStepper` and +:py:class:`~sympl.Stepper` objects. These types of objects take in the state and a timedelta object that indicates the time step, and return the next -model state. :py:class:`~sympl.TimeStepper` objects do this by wrapping -:py:class:`~sympl.Prognostic` objects, which calculate tendencies using the -state dictionary. We should note that the meaning of "Implicit" in Sympl is -slightly different than its traditional definition. Here an "Implicit" object is +model state. :py:class:`~sympl.PrognosticStepper` objects do this by wrapping +:py:class:`~sympl.PrognosticComponent` objects, which calculate tendencies using the +state dictionary. We should note that the meaning of "Stepper" in Sympl is +slightly different than its traditional definition. Here an "Stepper" object is one that calculates the new state directly from the current state, or any object that requires the timestep to calculate the new state, while -"Prognostic" objects are ones that calculate tendencies without using the -timestep. If a :py:class:`~sympl.TimeStepper` or :py:class:`~sympl.Implicit` +"PrognosticComponent" objects are ones that calculate tendencies without using the +timestep. If a :py:class:`~sympl.PrognosticStepper` or :py:class:`~sympl.Stepper` object needs to use multiple time steps in its calculation, it does so by storing states it was previously given until they are no longer needed. -The state is also calculated using :py:class:`~sympl.Diagnostic` objects which +The state is also calculated using :py:class:`~sympl.DiagnosticComponent` objects which determine diagnostic quantities at the current time from the current state, returning them in a new dictionary. This type of object is particularly useful if you want to write your own online diagnostics. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 6d1f8f2..58a5da6 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -137,26 +137,26 @@ Those are the "components": ]) implicit_dynamics = ImplicitDynamics() -:py:class:`~sympl.AdamsBashforth` is a :py:class:`~sympl.TimeStepper`, which is -created with a set of :py:class:`~sympl.Prognostic` components. -The :py:class:`~sympl.Prognostic` components we have here are ``Radiation``, +:py:class:`~sympl.AdamsBashforth` is a :py:class:`~sympl.PrognosticStepper`, which is +created with a set of :py:class:`~sympl.PrognosticComponent` components. +The :py:class:`~sympl.PrognosticComponent` components we have here are ``Radiation``, ``BoundaryLayer``, and ``DeepConvection``. Each of these carries information about what it takes as inputs and provides as outputs, and can be called with a model state to return tendencies for a set of quantities. The -:py:class:`~sympl.TimeStepper` uses this information to step the model state +:py:class:`~sympl.PrognosticStepper` uses this information to step the model state forward in time. The :py:class:`~sympl.UpdateFrequencyWrapper` applied to the ``Radiation`` object -is an object that acts like a :py:class:`~sympl.Prognostic` but only computes +is an object that acts like a :py:class:`~sympl.PrognosticComponent` but only computes its output if at least a certain amount of model time has passed since the last time the output was computed. Otherwise, it returns the last computed output. This is commonly used in atmospheric models to avoid doing radiation calculations (which are very expensive) every timestep, but it can be applied -to any Prognostic. +to any PrognosticComponent. -The :py:class:`ImplicitDynamics` class is a :py:class:`~sympl.Implicit` object, which -steps the model state forward in time in the same way that a :py:class:`~sympl.TimeStepper` -would, but doesn't use :py:class:`~sympl.Prognostic` objects in doing so. +The :py:class:`ImplicitDynamics` class is a :py:class:`~sympl.Stepper` object, which +steps the model state forward in time in the same way that a :py:class:`~sympl.PrognosticStepper` +would, but doesn't use :py:class:`~sympl.PrognosticComponent` objects in doing so. The Main Loop ------------- @@ -180,6 +180,6 @@ In the main loop, a series of component calls update the state, and the figure presented by ``plot_monitor`` is updated. The code is meant to be as self-explanatory as possible. It is necessary to manually set the time of the next state at the end of the loop. This is not done automatically by -:py:class:`~sympl.TimeStepper` and :py:class:`~sympl.Implicit` objects, because +:py:class:`~sympl.PrognosticStepper` and :py:class:`~sympl.Stepper` objects, because in many models you may want to update the state with multiple such objects in a sequence over the course of a single time step. diff --git a/docs/state.rst b/docs/state.rst index 7f543e4..e61f0b3 100644 --- a/docs/state.rst +++ b/docs/state.rst @@ -11,7 +11,7 @@ subtraction, and contains a helpful method for converting units. Any information about the grid the data is using that components need should be put as attributes in the ``attrs`` of the ``DataArray`` objects. Deciding on these attributes (if any) is mostly up to the component developers. However, -in order to use the TimeStepper objects and several helper functions from Sympl, +in order to use the PrognosticStepper objects and several helper functions from Sympl, it is required that a "units" attribute is present. .. _xarray: http://xarray.pydata.org/en/stable/ diff --git a/docs/timestepping.rst b/docs/timestepping.rst index 1f1a20d..ecf66ba 100644 --- a/docs/timestepping.rst +++ b/docs/timestepping.rst @@ -1,17 +1,17 @@ Timestepping ============ -:py:class:`~sympl.TimeStepper` objects use time derivatives from -:py:class:`~sympl.Prognostic` objects to step a model state forward in time. -They are initialized using any number of :py:class:`~sympl.Prognostic` objects. +:py:class:`~sympl.PrognosticStepper` objects use time derivatives from +:py:class:`~sympl.PrognosticComponent` objects to step a model state forward in time. +They are initialized using any number of :py:class:`~sympl.PrognosticComponent` objects. .. code-block:: python from sympl import AdamsBashforth time_stepper = AdamsBashforth(MyPrognostic(), MyOtherPrognostic()) -Once initialized, a :py:class:`~sympl.TimeStepper` object has a very similar -interface to the :py:class:`~sympl.Implicit` object. +Once initialized, a :py:class:`~sympl.PrognosticStepper` object has a very similar +interface to the :py:class:`~sympl.Stepper` object. .. code-block:: python @@ -29,32 +29,32 @@ and that they have been modified. In other words, ``state`` may be modified by this call. For instance, the time filtering necessary when using Leapfrog time stepping means the current model state has to be modified by the filter. -It is only after calling the :py:class:`~sympl.TimeStepper` and getting the +It is only after calling the :py:class:`~sympl.PrognosticStepper` and getting the diagnostics that you will have a complete state with all diagnostic quantities. This means you will sometimes want to pass ``state`` to your :py:class:`~sympl.Monitor` objects *after* calling -the :py:class:`~sympl.TimeStepper` and getting ``next_state``. +the :py:class:`~sympl.PrognosticStepper` and getting ``next_state``. -.. warning:: :py:class:`~sympl.TimeStepper` objects do not, and should not, +.. warning:: :py:class:`~sympl.PrognosticStepper` objects do not, and should not, update 'time' in the model state. -Keep in mind that for split-time models, multiple :py:class:`~sympl.TimeStepper` +Keep in mind that for split-time models, multiple :py:class:`~sympl.PrognosticStepper` objects might be called in in a single pass of the main loop. If each one updated ``state['time']``, the time would be moved forward more than it should. -For that reason, :py:class:`~sympl.TimeStepper` objects do not update +For that reason, :py:class:`~sympl.PrognosticStepper` objects do not update ``state['time']``. There are also -:py:class:`~sympl.Implicit` objects which evolve the state forward in time -without the use of Prognostic objects. These function exactly the same as a -:py:class:`~sympl.TimeStepper` once they are created, but do not accept -:py:class:`~sympl.Prognostic` objects when you create them. One example might +:py:class:`~sympl.Stepper` objects which evolve the state forward in time +without the use of PrognosticComponent objects. These function exactly the same as a +:py:class:`~sympl.PrognosticStepper` once they are created, but do not accept +:py:class:`~sympl.PrognosticComponent` objects when you create them. One example might be a component that condenses all supersaturated moisture over some time period. -:py:class:`~sympl.Implicit` objects are generally used for parameterizations +:py:class:`~sympl.Stepper` objects are generally used for parameterizations that work by determining the target model state in some way, or involve -limiters, and cannot be represented as a :py:class:`~sympl.Prognostic`. +limiters, and cannot be represented as a :py:class:`~sympl.PrognosticComponent`. -.. autoclass:: sympl.TimeStepper +.. autoclass:: sympl.PrognosticStepper :members: :special-members: :exclude-members: __weakref__,__metaclass__ diff --git a/docs/writing_components.rst b/docs/writing_components.rst index ddac639..234ca64 100644 --- a/docs/writing_components.rst +++ b/docs/writing_components.rst @@ -13,17 +13,17 @@ to talk about the parts of their code. Writing an Example ------------------ -Let's start with a Prognostic component which relaxes temperature towards some +Let's start with a PrognosticComponent component which relaxes temperature towards some target temperature. We'll go over the sections of this example step-by-step below. .. code-block:: python from sympl import ( - Prognostic, get_numpy_arrays_with_properties, + PrognosticComponent, get_numpy_arrays_with_properties, restore_data_arrays_with_properties) - class TemperatureRelaxation(Prognostic): + class TemperatureRelaxation(PrognosticComponent): input_properties = { 'air_temperature': { @@ -68,7 +68,7 @@ so that it can be found right away by anyone reading your code. .. code-block:: python from sympl import ( - Prognostic, get_numpy_arrays_with_properties, + PrognosticComponent, get_numpy_arrays_with_properties, restore_data_arrays_with_properties) Define an Object @@ -78,13 +78,13 @@ Once these are imported, there's this line: .. code-block:: python - class TemperatureRelaxation(Prognostic): + class TemperatureRelaxation(PrognosticComponent): This is the syntax for defining an object in Python. ``TemperatureRelaxation`` -will be the name of the new object. The :py:class:`~sympl.Prognostic` +will be the name of the new object. The :py:class:`~sympl.PrognosticComponent` in parentheses is telling Python that ``TemperatureRelaxation`` is a *subclass* of -:py:class:`~sympl.Prognostic`. This tells Sympl that it can expect your object -to behave like a :py:class:`~sympl.Prognostic`. +:py:class:`~sympl.PrognosticComponent`. This tells Sympl that it can expect your object +to behave like a :py:class:`~sympl.PrognosticComponent`. Define Attributes ***************** @@ -292,14 +292,14 @@ Using Tracers this, you probably don't. Sympl's base components have some features to automatically create tracer arrays -for use by dynamical components. If an :py:class:`~sympl.Implicit`, -:py:class:`~sympl.Prognostic`, or :py:class:`~sympl.ImplicitPrognostic` +for use by dynamical components. If an :py:class:`~sympl.Stepper`, +:py:class:`~sympl.PrognosticComponent`, or :py:class:`~sympl.ImplicitPrognosticComponent` component specifies ``uses_tracers = True`` and sets ``tracer_dims``, this feature is enabled. .. code-block:: python - class MyDynamicalCore(Implicit): + class MyDynamicalCore(Stepper): uses_tracers = True tracer_dims = ['tracer', '*', 'mid_levels'] @@ -313,9 +313,9 @@ tracer (you could call it "tracer number"). Once this feature is enabled, the ``state`` passed to ``array_call`` on the component will include a quantity called "tracers" with the dimensions specified by ``tracer_dims``. It will also be required that these tracers -are used in the output. For a :py:class:`~sympl.Implicit` component, "tracers" -must be present in the output state, and for a :py:class:`~sympl.Prognostic` or -:py:class:`~sympl.ImplicitPrognostic` component "tracers" must be present in +are used in the output. For a :py:class:`~sympl.Stepper` component, "tracers" +must be present in the output state, and for a :py:class:`~sympl.PrognosticComponent` or +:py:class:`~sympl.ImplicitPrognosticComponent` component "tracers" must be present in the tendencies, with the same dimensions as the input "tracers". On these latter two components, you should also specify a @@ -326,7 +326,7 @@ units of ``g m^-3 s^-1``. This value is set as "s" (or seconds) by default. .. code-block:: python - class MyDynamicalCore(Prognostic): + class MyDynamicalCore(PrognosticComponent): uses_tracers = True tracer_dims = ['tracer', '*', 'mid_levels'] diff --git a/sympl/__init__.py b/sympl/__init__.py index 7e7541c..26638c5 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- from ._core.base_components import ( - Prognostic, Diagnostic, Implicit, Monitor, ImplicitPrognostic + PrognosticComponent, DiagnosticComponent, Stepper, Monitor, ImplicitPrognosticComponent ) -from ._core.composite import PrognosticComposite, DiagnosticComposite, \ +from ._core.composite import PrognosticComponentComposite, DiagnosticComponentComposite, \ MonitorComposite -from ._core.timestepper import TimeStepper +from ._core.prognosticstepper import PrognosticStepper from ._components.timesteppers import AdamsBashforth, Leapfrog, SSPRungeKutta from ._core.exceptions import ( InvalidStateError, SharedKeyError, DependencyError, @@ -28,16 +28,16 @@ initialize_numpy_arrays_with_properties) from ._components import ( PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, - ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, + ConstantPrognosticComponent, ConstantDiagnosticComponent, RelaxationPrognosticComponent, TimeDifferencingWrapper) from ._core.wrappers import UpdateFrequencyWrapper, ScalingWrapper from ._core.time import datetime, timedelta __version__ = '0.3.2' __all__ = ( - Prognostic, Diagnostic, Implicit, Monitor, PrognosticComposite, - DiagnosticComposite, MonitorComposite, ImplicitPrognostic, - TimeStepper, Leapfrog, AdamsBashforth, SSPRungeKutta, + PrognosticComponent, DiagnosticComponent, Stepper, Monitor, PrognosticComponentComposite, + DiagnosticComponentComposite, MonitorComposite, ImplicitPrognosticComponent, + PrognosticStepper, Leapfrog, AdamsBashforth, SSPRungeKutta, InvalidStateError, SharedKeyError, DependencyError, InvalidPropertyDictError, ComponentExtraOutputError, ComponentMissingOutputError, @@ -53,7 +53,7 @@ initialize_numpy_arrays_with_properties, get_component_aliases, combine_component_properties, PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, - ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, + ConstantPrognosticComponent, ConstantDiagnosticComponent, RelaxationPrognosticComponent, UpdateFrequencyWrapper, ScalingWrapper, datetime, timedelta ) diff --git a/sympl/_components/__init__.py b/sympl/_components/__init__.py index 08ff312..931548c 100644 --- a/sympl/_components/__init__.py +++ b/sympl/_components/__init__.py @@ -1,11 +1,11 @@ from .netcdf import NetCDFMonitor, RestartMonitor from .plot import PlotFunctionMonitor from .basic import ( - ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, + ConstantPrognosticComponent, ConstantDiagnosticComponent, RelaxationPrognosticComponent, TimeDifferencingWrapper) __all__ = ( PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, - ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, + ConstantPrognosticComponent, ConstantDiagnosticComponent, RelaxationPrognosticComponent, TimeDifferencingWrapper) diff --git a/sympl/_components/basic.py b/sympl/_components/basic.py index f249133..a797c59 100644 --- a/sympl/_components/basic.py +++ b/sympl/_components/basic.py @@ -1,9 +1,9 @@ from .._core.array import DataArray -from .._core.base_components import ImplicitPrognostic, Prognostic, Diagnostic +from .._core.base_components import ImplicitPrognosticComponent, PrognosticComponent, DiagnosticComponent from .._core.units import unit_registry as ureg -class ConstantPrognostic(Prognostic): +class ConstantPrognosticComponent(PrognosticComponent): """ Prescribes constant tendencies provided at initialization. @@ -83,11 +83,11 @@ def __init__(self, tendencies, diagnostics=None, **kwargs): tendencies : dict A dictionary whose keys are strings indicating state quantities and values are the time derivative of those - quantities in units/second to be returned by this Prognostic. + quantities in units/second to be returned by this PrognosticComponent. diagnostics : dict, optional A dictionary whose keys are strings indicating state quantities and values are the value of those quantities - to be returned by this Prognostic. By default an empty dictionary + to be returned by this PrognosticComponent. By default an empty dictionary is used. input_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and @@ -118,7 +118,7 @@ def __init__(self, tendencies, diagnostics=None, **kwargs): self.__diagnostics = diagnostics.copy() else: self.__diagnostics = {} - super(ConstantPrognostic, self).__init__(**kwargs) + super(ConstantPrognosticComponent, self).__init__(**kwargs) def array_call(self, state): tendencies = {} @@ -130,7 +130,7 @@ def array_call(self, state): return tendencies, diagnostics -class ConstantDiagnostic(Diagnostic): +class ConstantDiagnosticComponent(DiagnosticComponent): """ Yields constant diagnostics provided at initialization. @@ -186,7 +186,7 @@ def __init__(self, diagnostics, **kwargs): A dictionary whose keys are strings indicating state quantities and values are the value of those quantities. The values in the dictionary will be returned when this - Diagnostic is called. + DiagnosticComponent is called. input_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and values are floats by which input values are scaled before being used @@ -201,7 +201,7 @@ def __init__(self, diagnostics, **kwargs): output was given. Otherwise, it would return that cached output. """ self.__diagnostics = diagnostics.copy() - super(ConstantDiagnostic, self).__init__(**kwargs) + super(ConstantDiagnosticComponent, self).__init__(**kwargs) def array_call(self, state): return_state = {} @@ -210,7 +210,7 @@ def array_call(self, state): return return_state -class RelaxationPrognostic(Prognostic): +class RelaxationPrognosticComponent(PrognosticComponent): r""" Applies Newtonian relaxation to a single quantity. @@ -319,7 +319,7 @@ def __init__(self, quantity_name, units, **kwargs): """ self._quantity_name = quantity_name self._units = units - super(RelaxationPrognostic, self).__init__(**kwargs) + super(RelaxationPrognosticComponent, self).__init__(**kwargs) def array_call(self, state): """ @@ -359,14 +359,14 @@ def array_call(self, state): return tendencies, {} -class TimeDifferencingWrapper(ImplicitPrognostic): +class TimeDifferencingWrapper(ImplicitPrognosticComponent): """ - Wraps an Implicit object and turns it into an ImplicitPrognostic by applying + Wraps an Stepper object and turns it into an ImplicitPrognosticComponent by applying simple first-order time differencing to determine tendencies. Example ------- - This how the wrapper should be used on an Implicit class + This how the wrapper should be used on an Stepper class called GridScaleCondensation. >>> component = TimeDifferencingWrapper(GridScaleCondensation()) @@ -388,14 +388,14 @@ def diagnostic_properties(self): def __init__(self, implicit, **kwargs): """ - Initializes the TimeDifferencingWrapper. Some kwargs of Implicit + Initializes the TimeDifferencingWrapper. Some kwargs of Stepper objects are not implemented, and should be applied instead on the - Implicit object which is wrapped by this one. + Stepper object which is wrapped by this one. Parameters ---------- - implicit: Implicit - An Implicit component to wrap. + implicit: Stepper + An Stepper component to wrap. """ if len(kwargs) > 0: raise TypeError('Received unexpected keyword argument {}'.format( diff --git a/sympl/_components/netcdf.py b/sympl/_components/netcdf.py index d5af713..34245c5 100644 --- a/sympl/_components/netcdf.py +++ b/sympl/_components/netcdf.py @@ -87,7 +87,7 @@ def store(self, state): Raises ------ InvalidStateError - If state is not a valid input for the Diagnostic instance. + If state is not a valid input for the DiagnosticComponent instance. """ if self._store_names is not None: name_list = set(state.keys()).intersection(self._store_names) diff --git a/sympl/_components/timesteppers.py b/sympl/_components/timesteppers.py index 098e599..440f258 100644 --- a/sympl/_components/timesteppers.py +++ b/sympl/_components/timesteppers.py @@ -1,11 +1,11 @@ -from .._core.timestepper import TimeStepper +from .._core.prognosticstepper import PrognosticStepper from .._core.array import DataArray from .._core.state import copy_untouched_quantities, add, multiply -class SSPRungeKutta(TimeStepper): +class SSPRungeKutta(PrognosticStepper): """ - A TimeStepper using the Strong Stability Preserving Runge-Kutta scheme, + A PrognosticStepper using the Strong Stability Preserving Runge-Kutta scheme, as in Numerical Methods for Fluid Dynamics by Dale Durran (2nd ed) and as proposed by Shu and Osher (1988). """ @@ -16,7 +16,7 @@ def __init__(self, *args, **kwargs): Args ---- - *args : Prognostic or ImplicitPrognostic + *args : PrognosticComponent or ImplicitPrognosticComponent Objects to call for tendencies when doing time stepping. stages: int, optional Number of stages to use. Should be 2 or 3. Default is 3. @@ -71,8 +71,8 @@ def _step_2_stages(self, state, timestep): return diagnostics, out_state -class AdamsBashforth(TimeStepper): - """A TimeStepper using the Adams-Bashforth scheme.""" +class AdamsBashforth(PrognosticStepper): + """A PrognosticStepper using the Adams-Bashforth scheme.""" def __init__(self, *args, **kwargs): """ @@ -80,7 +80,7 @@ def __init__(self, *args, **kwargs): Args ---- - *args : Prognostic or ImplicitPrognostic + *args : PrognosticComponent or ImplicitPrognosticComponent Objects to call for tendencies when doing time stepping. order : int, optional The order of accuracy to use. Must be between @@ -173,8 +173,8 @@ def convert_tendencies_units_for_state(tendencies, state): tendencies[quantity_name] = tendencies[quantity_name].to_units(desired_units) -class Leapfrog(TimeStepper): - """A TimeStepper using the Leapfrog scheme. +class Leapfrog(PrognosticStepper): + """A PrognosticStepper using the Leapfrog scheme. This scheme calculates the values at time $t_{n+1}$ using the derivatives at $t_{n}$ and values at @@ -190,7 +190,7 @@ def __init__(self, *args, **kwargs): Args ---- - *args : Prognostic or ImplicitPrognostic + *args : PrognosticComponent or ImplicitPrognosticComponent Objects to call for tendencies when doing time stepping. asselin_strength : float, optional The filter parameter used to determine the strength @@ -238,7 +238,7 @@ def _call(self, state, timestep): Raises ------ SharedKeyError - If a Diagnostic object has an output that is + If a DiagnosticComponent object has an output that is already in the state at the start of the timestep. ValueError If the timestep is not the same as the last time diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 5a03cbd..bb110e9 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -26,11 +26,11 @@ def apply_scale_factors(array_state, scale_factors): def is_component_class(cls): - return any(issubclass(cls, cls2) for cls2 in (Implicit, Prognostic, ImplicitPrognostic, Diagnostic)) + return any(issubclass(cls, cls2) for cls2 in (Stepper, PrognosticComponent, ImplicitPrognosticComponent, DiagnosticComponent)) def is_component_base_class(cls): - return cls in (Implicit, Prognostic, ImplicitPrognostic, Diagnostic) + return cls in (Stepper, PrognosticComponent, ImplicitPrognosticComponent, DiagnosticComponent) def get_kwarg_defaults(func): @@ -226,7 +226,7 @@ def _check_missing_tendencies(self, tendency_dict): if len(missing_tendencies) > 0: raise ComponentMissingOutputError( 'Component {} did not compute tendencies for {}'.format( - self.__class__.__name__, ', '.join(missing_tendencies))) + self.component.__class__.__name__, ', '.join(missing_tendencies))) def _check_extra_tendencies(self, tendency_dict): wanted_set = set() @@ -238,7 +238,7 @@ def _check_extra_tendencies(self, tendency_dict): raise ComponentExtraOutputError( 'Component {} computed tendencies for {} which are not in ' 'tendency_properties'.format( - self.__class__.__name__, ', '.join(extra_tendencies))) + self.component.__class__.__name__, ', '.join(extra_tendencies))) def check_tendencies(self, tendency_dict): self._check_missing_tendencies(tendency_dict) @@ -265,10 +265,10 @@ def __init__(self, component): for name, properties in component.diagnostic_properties.items(): if 'units' not in properties.keys(): raise InvalidPropertyDictError( - 'Diagnostic properties do not have units defined for {}'.format(name)) + 'DiagnosticComponent properties do not have units defined for {}'.format(name)) if 'dims' not in properties.keys() and name not in component.input_properties.keys(): raise InvalidPropertyDictError( - 'Diagnostic properties do not have dims defined for {}'.format(name) + 'DiagnosticComponent properties do not have dims defined for {}'.format(name) ) incompatible_name = get_name_with_incompatible_units( self.component.input_properties, self.component.diagnostic_properties) @@ -385,7 +385,7 @@ def _check_missing_outputs(self, outputs_dict): if len(missing_outputs) > 0: raise ComponentMissingOutputError( 'Component {} did not compute output(s) {}'.format( - self.__class__.__name__, ', '.join(missing_outputs))) + self.component.__class__.__name__, ', '.join(missing_outputs))) def _check_extra_outputs(self, outputs_dict): wanted_set = set() @@ -397,7 +397,7 @@ def _check_extra_outputs(self, outputs_dict): raise ComponentExtraOutputError( 'Component {} computed output(s) {} which are not in ' 'output_properties'.format( - self.__class__.__name__, ', '.join(extra_outputs))) + self.component.__class__.__name__, ', '.join(extra_outputs))) def check_outputs(self, output_dict): self._check_missing_outputs(output_dict) @@ -405,7 +405,7 @@ def check_outputs(self, output_dict): @add_metaclass(ComponentMeta) -class Implicit(object): +class Stepper(object): """ Attributes ---------- @@ -457,7 +457,7 @@ def output_properties(self): def __str__(self): return ( - 'instance of {}(Implicit)\n' + 'instance of {}(Stepper)\n' ' inputs: {}\n' ' outputs: {}\n' ' diagnostics: {}'.format( @@ -481,7 +481,7 @@ def __repr__(self): def __init__(self, tendencies_in_diagnostics=False, name=None): """ - Initializes the Implicit object. + Initializes the Stepper object. Args ---- @@ -495,8 +495,8 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): lowercase is used. """ self._tendencies_in_diagnostics = tendencies_in_diagnostics - self.name = name or self.__class__.__name__.lower() - super(Implicit, self).__init__() + self.name = name or self.__class__.__name__ + super(Stepper, self).__init__() self._input_checker = InputChecker(self) self._diagnostic_checker = DiagnosticChecker(self) self._output_checker = OutputChecker(self) @@ -512,7 +512,7 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): prepend_tracers = getattr(self, 'prepend_tracers', None) self._tracer_packer = TracerPacker( self, self.tracer_dims, prepend_tracers=prepend_tracers) - super(Implicit, self).__init__() + super(Stepper, self).__init__() def _insert_tendency_properties(self): added_names = [] @@ -604,7 +604,7 @@ def __call__(self, state, timestep): KeyError If a required quantity is missing from the state. InvalidStateError - If state is not a valid input for the Implicit instance + If state is not a valid input for the Stepper instance for other reasons. """ self._check_self_is_initialized() @@ -668,7 +668,7 @@ def array_call(self, state, timestep): @add_metaclass(ComponentMeta) -class Prognostic(object): +class PrognosticComponent(object): """ Attributes ---------- @@ -711,7 +711,7 @@ def diagnostic_properties(self): def __str__(self): return ( - 'instance of {}(Prognostic)\n' + 'instance of {}(PrognosticComponent)\n' ' inputs: {}\n' ' tendencies: {}\n' ' diagnostics: {}'.format( @@ -735,7 +735,7 @@ def __repr__(self): def __init__(self, tendencies_in_diagnostics=False, name=None): """ - Initializes the Implicit object. + Initializes the Stepper object. Args ---- @@ -766,7 +766,7 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): self._tracer_packer = TracerPacker( self, self.tracer_dims, prepend_tracers=prepend_tracers) self.__initialized = True - super(Prognostic, self).__init__() + super(PrognosticComponent, self).__init__() @property def tendencies_in_diagnostics(self): @@ -831,7 +831,7 @@ def __call__(self, state): KeyError If a required quantity is missing from the state. InvalidStateError - If state is not a valid input for the Prognostic instance. + If state is not a valid input for the PrognosticComponent instance. """ self._check_self_is_initialized() self._input_checker.check_inputs(state) @@ -892,7 +892,7 @@ def array_call(self, state): @add_metaclass(ComponentMeta) -class ImplicitPrognostic(object): +class ImplicitPrognosticComponent(object): """ Attributes ---------- @@ -935,7 +935,7 @@ def diagnostic_properties(self): def __str__(self): return ( - 'instance of {}(Prognostic)\n' + 'instance of {}(PrognosticComponent)\n' ' inputs: {}\n' ' tendencies: {}\n' ' diagnostics: {}'.format( @@ -957,7 +957,7 @@ def __repr__(self): def __init__(self, tendencies_in_diagnostics=False, name=None): """ - Initializes the Implicit object. + Initializes the Stepper object. Args ---- @@ -987,7 +987,7 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): self._tracer_packer = TracerPacker( self, self.tracer_dims, prepend_tracers=prepend_tracers) self.__initialized = True - super(ImplicitPrognostic, self).__init__() + super(ImplicitPrognosticComponent, self).__init__() @property def tendencies_in_diagnostics(self): @@ -1054,7 +1054,7 @@ def __call__(self, state, timestep): KeyError If a required quantity is missing from the state. InvalidStateError - If state is not a valid input for the Prognostic instance. + If state is not a valid input for the PrognosticComponent instance. """ self._check_self_is_initialized() self._input_checker.check_inputs(state) @@ -1117,7 +1117,7 @@ def array_call(self, state, timestep): @add_metaclass(ComponentMeta) -class Diagnostic(object): +class DiagnosticComponent(object): """ Attributes ---------- @@ -1141,7 +1141,7 @@ def diagnostic_properties(self): def __str__(self): return ( - 'instance of {}(Diagnostic)\n' + 'instance of {}(DiagnosticComponent)\n' ' inputs: {}\n' ' diagnostics: {}'.format( self.__class__, self.inputs, self.diagnostics) @@ -1162,12 +1162,12 @@ def __repr__(self): def __init__(self): """ - Initializes the Implicit object. + Initializes the Stepper object. """ self._input_checker = InputChecker(self) self._diagnostic_checker = DiagnosticChecker(self) self.__initialized = True - super(Diagnostic, self).__init__() + super(DiagnosticComponent, self).__init__() def _check_self_is_initialized(self): try: @@ -1205,7 +1205,7 @@ def __call__(self, state): KeyError If a required quantity is missing from the state. InvalidStateError - If state is not a valid input for the Prognostic instance. + If state is not a valid input for the PrognosticComponent instance. """ self._check_self_is_initialized() self._input_checker.check_inputs(state) @@ -1272,5 +1272,5 @@ def store(self, state): Raises ------ InvalidStateError - If state is not a valid input for the Diagnostic instance. + If state is not a valid input for the DiagnosticComponent instance. """ diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 4f75d8f..6eefb1e 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -1,4 +1,4 @@ -from .base_components import Prognostic, Diagnostic, Monitor, ImplicitPrognostic +from .base_components import PrognosticComponent, DiagnosticComponent, Monitor, ImplicitPrognosticComponent from .util import ( update_dict_by_adding_another, ensure_no_shared_keys, combine_component_properties) @@ -99,11 +99,11 @@ def ensure_components_have_class(components, component_class): component_class, type(component))) -class PrognosticComposite( +class PrognosticComponentComposite( ComponentComposite, InputPropertiesCompositeMixin, - DiagnosticPropertiesCompositeMixin, Prognostic): + DiagnosticPropertiesCompositeMixin, PrognosticComponent): - component_class = Prognostic + component_class = PrognosticComponent @property def tendency_properties(self): @@ -126,7 +126,7 @@ def __init__(self, *args): output quantity, and their dimensions or units are incompatible with one another. """ - super(PrognosticComposite, self).__init__(*args) + super(PrognosticComponentComposite, self).__init__(*args) self.input_properties self.tendency_properties self.diagnostic_properties @@ -156,7 +156,7 @@ def __call__(self, state): KeyError If a required quantity is missing from the state. InvalidStateError - If state is not a valid input for a Prognostic instance. + If state is not a valid input for a PrognosticComponent instance. """ return_tendencies = {} return_diagnostics = {} @@ -170,11 +170,11 @@ def array_call(self, state): raise NotImplementedError() -class ImplicitPrognosticComposite( +class ImplicitPrognosticComponentComposite( ComponentComposite, InputPropertiesCompositeMixin, - DiagnosticPropertiesCompositeMixin, ImplicitPrognostic): + DiagnosticPropertiesCompositeMixin, ImplicitPrognosticComponent): - component_class = (Prognostic, ImplicitPrognostic) + component_class = (PrognosticComponent, ImplicitPrognosticComponent) @property def tendency_properties(self): @@ -197,7 +197,7 @@ def __init__(self, *args): output quantity, and their dimensions or units are incompatible with one another. """ - super(ImplicitPrognosticComposite, self).__init__(*args) + super(ImplicitPrognosticComponentComposite, self).__init__(*args) self.input_properties self.tendency_properties self.diagnostic_properties @@ -227,14 +227,14 @@ def __call__(self, state, timestep): KeyError If a required quantity is missing from the state. InvalidStateError - If state is not a valid input for a Prognostic instance. + If state is not a valid input for a PrognosticComponent instance. """ return_tendencies = {} return_diagnostics = {} for prognostic in self.component_list: - if isinstance(prognostic, ImplicitPrognostic): + if isinstance(prognostic, ImplicitPrognosticComponent): tendencies, diagnostics = prognostic(state, timestep) - elif isinstance(prognostic, Prognostic): + elif isinstance(prognostic, PrognosticComponent): tendencies, diagnostics = prognostic(state) update_dict_by_adding_another(return_tendencies, tendencies) return_diagnostics.update(diagnostics) @@ -244,11 +244,11 @@ def array_call(self, state): raise NotImplementedError() -class DiagnosticComposite( +class DiagnosticComponentComposite( ComponentComposite, InputPropertiesCompositeMixin, - DiagnosticPropertiesCompositeMixin, Diagnostic): + DiagnosticPropertiesCompositeMixin, DiagnosticComponent): - component_class = Diagnostic + component_class = DiagnosticComponent def __call__(self, state): """ @@ -271,7 +271,7 @@ def __call__(self, state): KeyError If a required quantity is missing from the state. InvalidStateError - If state is not a valid input for a Diagnostic instance. + If state is not a valid input for a DiagnosticComponent instance. """ return_diagnostics = {} for diagnostic_component in self.component_list: diff --git a/sympl/_core/timestepper.py b/sympl/_core/prognosticstepper.py similarity index 82% rename from sympl/_core/timestepper.py rename to sympl/_core/prognosticstepper.py index 7008df0..f81db45 100644 --- a/sympl/_core/timestepper.py +++ b/sympl/_core/prognosticstepper.py @@ -1,17 +1,17 @@ import abc -from .composite import ImplicitPrognosticComposite +from .composite import ImplicitPrognosticComponentComposite from .time import timedelta from .util import combine_component_properties, combine_properties from .units import clean_units from .state import copy_untouched_quantities -from .base_components import ImplicitPrognostic +from .base_components import ImplicitPrognosticComponent, Stepper import warnings -class TimeStepper(object): +class PrognosticStepper(Stepper): """An object which integrates model state forward in time. - It uses Prognostic and Diagnostic objects to update the current model state + It uses PrognosticComponent and DiagnosticComponent objects to update the current model state with diagnostics, and to return the model state at the next timestep. Attributes @@ -26,13 +26,13 @@ class TimeStepper(object): for the new state are returned when the object is called, and values are dictionaries which indicate 'dims' and 'units'. - prognostic : ImplicitPrognosticComposite - A composite of the Prognostic and ImplicitPrognostic objects used by - the TimeStepper. - prognostic_list: list of Prognostic and ImplicitPrognostic - A list of Prognostic objects called by the TimeStepper. These should + prognostic : ImplicitPrognosticComponentComposite + A composite of the PrognosticComponent and ImplicitPrognostic objects used by + the PrognosticStepper. + prognostic_list: list of PrognosticComponent and ImplicitPrognosticComponent + A list of PrognosticComponent objects called by the PrognosticStepper. These should be referenced when determining what inputs are necessary for the - TimeStepper. + PrognosticStepper. tendencies_in_diagnostics : bool A boolean indicating whether this object will put tendencies of quantities in its diagnostic output. @@ -93,8 +93,8 @@ def _tendency_properties(self): def __str__(self): return ( - 'instance of {}(TimeStepper)\n' - ' Prognostic components: {}'.format(self.prognostic_list) + 'instance of {}(PrognosticStepper)\n' + ' PrognosticComponent components: {}'.format(self.prognostic_list) ) def __repr__(self): @@ -110,13 +110,16 @@ def __repr__(self): self._making_repr = False return return_value + def array_call(self, state, timestep): + raise NotImplementedError('PrognosticStepper objects do not implement array_call') + def __init__(self, *args, **kwargs): """ - Initialize the TimeStepper. + Initialize the PrognosticStepper. Parameters ---------- - *args : Prognostic or ImplicitPrognostic + *args : PrognosticComponent or ImplicitPrognosticComponent Objects to call for tendencies when doing time stepping. tendencies_in_diagnostics : bool, optional A boolean indicating whether this object will put tendencies of @@ -126,26 +129,19 @@ def __init__(self, *args, **kwargs): A label to be used for this object, for example as would be used for Y in the name "X_tendency_from_Y". By default the class name is used. """ - self.name = kwargs.pop('name', self.__class__.__name__) - tendencies_in_diagnostics = kwargs.pop('tendencies_in_diagnostics', False) - if len(kwargs) > 0: - raise TypeError( - "TimeStepper.__init__ got an unexpected keyword argument '{}'".format( - kwargs.popitem()[0])) if len(args) == 1 and isinstance(args[0], list): warnings.warn( 'TimeSteppers should be given individual Prognostics rather ' 'than a list, and will not accept lists in a later version.', DeprecationWarning) args = args[0] - self._tendencies_in_diagnostics = tendencies_in_diagnostics - # warnings.simplefilter('always') - if any(isinstance(a, ImplicitPrognostic) for a in args): + if any(isinstance(a, ImplicitPrognosticComponent) for a in args): warnings.warn( - 'Using an ImplicitPrognostic in sympl TimeStepper objects may ' + 'Using an ImplicitPrognosticComponent in sympl PrognosticStepper objects may ' 'lead to scientifically invalid results. Make sure the component ' - 'follows the same numerical assumptions as the TimeStepper used.') - self.prognostic = ImplicitPrognosticComposite(*args) + 'follows the same numerical assumptions as the PrognosticStepper used.') + self.prognostic = ImplicitPrognosticComponentComposite(*args) + super(PrognosticStepper, self).__init__(**kwargs) @property def prognostic_list(self): @@ -192,10 +188,10 @@ def _insert_tendencies_to_diagnostics( tendency_name = self._get_tendency_name(name) if tendency_name in diagnostics.keys(): raise RuntimeError( - 'A Prognostic has output tendencies as a diagnostic and has' + 'A PrognosticComponent has output tendencies as a diagnostic and has' ' caused a name clash when trying to do so from this ' - 'TimeStepper ({}). You must disable ' - 'tendencies_in_diagnostics for this TimeStepper.'.format( + 'PrognosticStepper ({}). You must disable ' + 'tendencies_in_diagnostics for this PrognosticStepper.'.format( tendency_name)) base_units = input_properties[name]['units'] diagnostics[tendency_name] = ( @@ -209,7 +205,6 @@ def _insert_tendencies_to_diagnostics( diagnostics[tendency_name].attrs['units'] = '{} {}^-1'.format( base_units, self.time_unit_name) - @abc.abstractmethod def _call(self, state, timestep): """ Retrieves any diagnostics and returns a new state corresponding @@ -229,3 +224,4 @@ def _call(self, state, timestep): new_state : dict The model state at the next timestep. """ + raise NotImplementedError() diff --git a/sympl/_core/state.py b/sympl/_core/state.py index 97ba843..db3bead 100644 --- a/sympl/_core/state.py +++ b/sympl/_core/state.py @@ -307,7 +307,7 @@ def check_array_shape(out_dims, raw_array, name, dim_lengths): def restore_data_arrays_with_properties( raw_arrays, output_properties, input_state, input_properties, - ignore_names=None): + ignore_names=None, ignore_missing=False): """ Parameters ---------- @@ -328,9 +328,12 @@ def restore_data_arrays_with_properties( with input properties for those quantities. The property "dims" must be present, indicating the dimensions that the quantity was transformed to when taken as input to a component. - ignore_names : iterable of str + ignore_names : iterable of str, optional Names to ignore when encountered in output_properties, will not be included in the returned dictionary. + ignore_missing : bool, optional + If True, ignore any values in output_properties not present in + raw_arrays rather than raising an exception. Default is False. Returns ------- @@ -349,6 +352,8 @@ def restore_data_arrays_with_properties( raw_arrays = raw_arrays.copy() if ignore_names is None: ignore_names = [] + if ignore_missing: + ignore_names = set(output_properties.keys()).difference(raw_arrays.keys()).union(ignore_names) wildcard_names, dim_lengths = get_wildcard_matches_and_dim_lengths( input_state, input_properties) ensure_values_are_arrays(raw_arrays) diff --git a/sympl/_core/tracers.py b/sympl/_core/tracers.py index c7056c7..51a28b8 100644 --- a/sympl/_core/tracers.py +++ b/sympl/_core/tracers.py @@ -83,8 +83,8 @@ def __init__(self, component, tracer_dims, prepend_tracers=None): self.component = component else: raise TypeError( - 'Expected a component object subclassing type Implicit, ' - 'ImplicitPrognostic, or Prognostic but received component of ' + 'Expected a component object subclassing type Stepper, ' + 'ImplicitPrognosticComponent, or PrognosticComponent but received component of ' 'type {}'.format(component.__class__.__name__)) for name, units in self._prepend_tracers: if name not in _tracer_unit_dict.keys(): diff --git a/sympl/_core/util.py b/sympl/_core/util.py index d78ec00..460f47a 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -420,8 +420,8 @@ def combine_properties(property_list, input_properties=None): def get_component_aliases(*args): """ - Returns aliases for variables in the properties of Components (Prognostic, - Diagnostic, Implicit, and ImplicitPrognostic objects). + Returns aliases for variables in the properties of Components (PrognosticComponent, + DiagnosticComponent, Stepper, and ImplicitPrognosticComponent objects). If multiple aliases are present for the same variable, the following properties have priority in descending order: input, output, diagnostic, diff --git a/sympl/_core/wrappers.py b/sympl/_core/wrappers.py index f7104e1..57c850c 100644 --- a/sympl/_core/wrappers.py +++ b/sympl/_core/wrappers.py @@ -1,5 +1,5 @@ from .._core.base_components import ( - Prognostic, Diagnostic, ImplicitPrognostic, Implicit + PrognosticComponent, DiagnosticComponent, ImplicitPrognosticComponent, Stepper ) @@ -10,7 +10,7 @@ class ScalingWrapper(object): Example ------- - This is how the ScalingWrapper can be used to wrap a Prognostic. + This is how the ScalingWrapper can be used to wrap a PrognosticComponent. >>> scaled_component = ScalingWrapper( >>> RRTMRadiation(), >>> input_scale_factors = { @@ -30,7 +30,7 @@ def __init__(self, Args ---- - component : Prognostic, Implicit, Diagnostic, ImplicitPrognostic + component : PrognosticComponent, Stepper, DiagnosticComponent, ImplicitPrognosticComponent The component to be wrapped. input_scale_factors : dict a dictionary whose keys are the inputs that will be scaled @@ -53,17 +53,17 @@ def __init__(self, Raises ------ TypeError - The component is not of type Implicit or Prognostic. + The component is not of type Stepper or PrognosticComponent. ValueError The keys in the scale factors do not correspond to valid input/output/tendency for this component. """ if not any( isinstance(component, t) for t in [ - Diagnostic, Prognostic, ImplicitPrognostic, Implicit]): + DiagnosticComponent, PrognosticComponent, ImplicitPrognosticComponent, Stepper]): raise TypeError( - 'component must be a component type (Diagnostic, Prognostic, ' - 'ImplicitPrognostic, or Implicit)' + 'component must be a component type (DiagnosticComponent, PrognosticComponent, ' + 'ImplicitPrognosticComponent, or Stepper)' ) self._component = component @@ -147,9 +147,9 @@ def __call__(self, state, timestep=None): else: scaled_state[input_field] = state[input_field] - if isinstance(self._component, Implicit): + if isinstance(self._component, Stepper): if timestep is None: - raise TypeError('Must give timestep to call Implicit.') + raise TypeError('Must give timestep to call Stepper.') diagnostics, new_state = self._component(scaled_state, timestep) for name in self._output_scale_factors.keys(): scale_factor = self._output_scale_factors[name] @@ -158,7 +158,7 @@ def __call__(self, state, timestep=None): scale_factor = self._diagnostic_scale_factors[name] diagnostics[name] *= float(scale_factor) return diagnostics, new_state - elif isinstance(self._component, Prognostic): + elif isinstance(self._component, PrognosticComponent): tendencies, diagnostics = self._component(scaled_state) for tend_field in self._tendency_scale_factors.keys(): scale_factor = self._tendency_scale_factors[tend_field] @@ -167,9 +167,9 @@ def __call__(self, state, timestep=None): scale_factor = self._diagnostic_scale_factors[name] diagnostics[name] *= float(scale_factor) return tendencies, diagnostics - elif isinstance(self._component, ImplicitPrognostic): + elif isinstance(self._component, ImplicitPrognosticComponent): if timestep is None: - raise TypeError('Must give timestep to call ImplicitPrognostic.') + raise TypeError('Must give timestep to call ImplicitPrognosticComponent.') tendencies, diagnostics = self._component(scaled_state, timestep) for tend_field in self._tendency_scale_factors.keys(): scale_factor = self._tendency_scale_factors[tend_field] @@ -178,7 +178,7 @@ def __call__(self, state, timestep=None): scale_factor = self._diagnostic_scale_factors[name] diagnostics[name] *= float(scale_factor) return tendencies, diagnostics - elif isinstance(self._component, Diagnostic): + elif isinstance(self._component, DiagnosticComponent): diagnostics = self._component(scaled_state) for name in self._diagnostic_scale_factors.keys(): scale_factor = self._diagnostic_scale_factors[name] @@ -197,7 +197,7 @@ class UpdateFrequencyWrapper(object): Example ------- - This how the wrapper should be used on a fictional Prognostic class + This how the wrapper should be used on a fictional PrognosticComponent class called MyPrognostic. >>> from datetime import timedelta >>> prognostic = UpdateFrequencyWrapper(MyPrognostic(), timedelta(hours=1)) @@ -209,7 +209,7 @@ def __init__(self, component, update_timedelta): Args ---- - component : Prognostic, Implicit, Diagnostic, ImplicitPrognostic + component : PrognosticComponent, Stepper, DiagnosticComponent, ImplicitPrognosticComponent The component to be wrapped. update_timedelta : timedelta The amount that state['time'] must differ from when output diff --git a/tests/test_base_components.py b/tests/test_base_components.py index 3f2e729..01dae41 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -3,7 +3,7 @@ import numpy as np import unittest from sympl import ( - Prognostic, Diagnostic, Monitor, Implicit, ImplicitPrognostic, + PrognosticComponent, DiagnosticComponent, Monitor, Stepper, ImplicitPrognosticComponent, datetime, timedelta, DataArray, InvalidPropertyDictError, ComponentMissingOutputError, ComponentExtraOutputError, InvalidStateError @@ -14,7 +14,7 @@ def same_list(list1, list2): [item in list2 for item in list1] + [item in list1 for item in list2])) -class MockPrognostic(Prognostic): +class MockPrognosticComponent(PrognosticComponent): input_properties = None diagnostic_properties = None @@ -30,7 +30,7 @@ def __init__( self.tendency_output = tendency_output self.times_called = 0 self.state_given = None - super(MockPrognostic, self).__init__(**kwargs) + super(MockPrognosticComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -38,7 +38,7 @@ def array_call(self, state): return self.tendency_output, self.diagnostic_output -class MockImplicitPrognostic(ImplicitPrognostic): +class MockImplicitPrognosticComponent(ImplicitPrognosticComponent): input_properties = None diagnostic_properties = None @@ -55,7 +55,7 @@ def __init__( self.times_called = 0 self.state_given = None self.timestep_given = None - super(MockImplicitPrognostic, self).__init__(**kwargs) + super(MockImplicitPrognosticComponent, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -64,7 +64,7 @@ def array_call(self, state, timestep): return self.tendency_output, self.diagnostic_output -class MockDiagnostic(Diagnostic): +class MockDiagnosticComponent(DiagnosticComponent): input_properties = None diagnostic_properties = None @@ -77,7 +77,7 @@ def __init__( self.diagnostic_output = diagnostic_output self.times_called = 0 self.state_given = None - super(MockDiagnostic, self).__init__(**kwargs) + super(MockDiagnosticComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -85,7 +85,7 @@ def array_call(self, state): return self.diagnostic_output -class MockImplicit(Implicit): +class MockStepper(Stepper): input_properties = None diagnostic_properties = None @@ -103,7 +103,7 @@ def __init__( self.times_called = 0 self.state_given = None self.timestep_given = None - super(MockImplicit, self).__init__(**kwargs) + super(MockStepper, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -117,7 +117,7 @@ class MockMonitor(Monitor): def store(self, state): return -class BadMockPrognostic(Prognostic): +class BadMockPrognosticComponent(PrognosticComponent): input_properties = {} tendency_properties = {} @@ -130,7 +130,7 @@ def array_call(self, state): return {}, {} -class BadMockImplicitPrognostic(ImplicitPrognostic): +class BadMockImplicitPrognosticComponent(ImplicitPrognosticComponent): input_properties = {} tendency_properties = {} @@ -143,7 +143,7 @@ def array_call(self, state, timestep): return {}, {} -class BadMockDiagnostic(Diagnostic): +class BadMockDiagnosticComponent(DiagnosticComponent): input_properties = {} diagnostic_properties = {} @@ -155,7 +155,7 @@ def array_call(self, state): return {} -class BadMockImplicit(Implicit): +class BadMockStepper(Stepper): input_properties = {} diagnostic_properties = {} @@ -733,7 +733,7 @@ def test_raises_when_extraneous_diagnostic_given(self): class PrognosticTests(unittest.TestCase, InputTestBase): - component_class = MockPrognostic + component_class = MockPrognosticComponent def call_component(self, component, state): return component(state) @@ -742,7 +742,7 @@ def get_component( self, input_properties=None, tendency_properties=None, diagnostic_properties=None, tendency_output=None, diagnostic_output=None): - return MockPrognostic( + return MockPrognosticComponent( input_properties=input_properties or {}, tendency_properties=tendency_properties or {}, diagnostic_properties=diagnostic_properties or {}, @@ -758,7 +758,7 @@ def test_raises_on_tendency_properties_of_wrong_type(self): self.get_component(tendency_properties=({},)) def test_cannot_use_bad_component(self): - component = BadMockPrognostic() + component = BadMockPrognosticComponent() with self.assertRaises(RuntimeError): self.call_component(component, {'time': timedelta(0)}) @@ -775,7 +775,7 @@ def array_call(self): pass instance = MyPrognostic() - assert isinstance(instance, Prognostic) + assert isinstance(instance, PrognosticComponent) def test_tendency_raises_when_units_incompatible_with_input(self): input_properties = { @@ -791,7 +791,7 @@ def test_tendency_raises_when_units_incompatible_with_input(self): ) def test_two_components_are_not_instances_of_each_other(self): - class MyPrognostic1(Prognostic): + class MyPrognosticComponent1(PrognosticComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -800,7 +800,7 @@ class MyPrognostic1(Prognostic): def array_call(self, state): pass - class MyPrognostic2(Prognostic): + class MyPrognosticComponent2(PrognosticComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -809,10 +809,10 @@ class MyPrognostic2(Prognostic): def array_call(self, state): pass - prog1 = MyPrognostic1() - prog2 = MyPrognostic2() - assert not isinstance(prog1, MyPrognostic2) - assert not isinstance(prog2, MyPrognostic1) + prog1 = MyPrognosticComponent1() + prog2 = MyPrognosticComponent2() + assert not isinstance(prog1, MyPrognosticComponent2) + assert not isinstance(prog2, MyPrognosticComponent1) def test_ducktype_not_instance_of_subclass(self): class MyPrognostic1(object): @@ -826,7 +826,7 @@ def __init__(self): def array_call(self, state): pass - class MyPrognostic2(Prognostic): + class MyPrognosticComponent2(PrognosticComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -836,7 +836,7 @@ def array_call(self, state): pass prog1 = MyPrognostic1() - assert not isinstance(prog1, MyPrognostic2) + assert not isinstance(prog1, MyPrognosticComponent2) def test_empty_prognostic(self): prognostic = self.component_class({}, {}, {}, {}, {}) @@ -1289,7 +1289,7 @@ def test_tendencies_in_diagnostics_one_tendency_with_component_name(self): class ImplicitPrognosticTests(PrognosticTests): - component_class = MockImplicitPrognostic + component_class = MockImplicitPrognosticComponent def call_component(self, component, state): return component(state, timedelta(seconds=1)) @@ -1298,7 +1298,7 @@ def get_component( self, input_properties=None, tendency_properties=None, diagnostic_properties=None, tendency_output=None, diagnostic_output=None): - return MockImplicitPrognostic( + return MockImplicitPrognosticComponent( input_properties=input_properties or {}, tendency_properties=tendency_properties or {}, diagnostic_properties=diagnostic_properties or {}, @@ -1307,7 +1307,7 @@ def get_component( ) def test_cannot_use_bad_component(self): - component = BadMockImplicitPrognostic() + component = BadMockImplicitPrognosticComponent() with self.assertRaises(RuntimeError): self.call_component(component, {'time': timedelta(0)}) @@ -1324,10 +1324,10 @@ def array_call(self, state, timestep): pass instance = MyImplicitPrognostic() - assert isinstance(instance, ImplicitPrognostic) + assert isinstance(instance, ImplicitPrognosticComponent) def test_two_components_are_not_instances_of_each_other(self): - class MyImplicitPrognostic1(ImplicitPrognostic): + class MyImplicitPrognosticComponent1(ImplicitPrognosticComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -1337,7 +1337,7 @@ class MyImplicitPrognostic1(ImplicitPrognostic): def array_call(self, state, timestep): pass - class MyImplicitPrognostic2(ImplicitPrognostic): + class MyImplicitPrognosticComponent2(ImplicitPrognosticComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -1347,10 +1347,10 @@ class MyImplicitPrognostic2(ImplicitPrognostic): def array_call(self, state): pass - prog1 = MyImplicitPrognostic1() - prog2 = MyImplicitPrognostic2() - assert not isinstance(prog1, MyImplicitPrognostic2) - assert not isinstance(prog2, MyImplicitPrognostic1) + prog1 = MyImplicitPrognosticComponent1() + prog2 = MyImplicitPrognosticComponent2() + assert not isinstance(prog1, MyImplicitPrognosticComponent2) + assert not isinstance(prog2, MyImplicitPrognosticComponent1) def test_ducktype_not_instance_of_subclass(self): class MyImplicitPrognostic1(object): @@ -1364,7 +1364,7 @@ def __init__(self): def array_call(self, state, timestep): pass - class MyImplicitPrognostic2(ImplicitPrognostic): + class MyImplicitPrognosticComponent2(ImplicitPrognosticComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -1375,10 +1375,10 @@ def array_call(self, state): pass prog1 = MyImplicitPrognostic1() - assert not isinstance(prog1, MyImplicitPrognostic2) + assert not isinstance(prog1, MyImplicitPrognosticComponent2) def test_subclass_is_not_prognostic(self): - class MyImplicitPrognostic(ImplicitPrognostic): + class MyImplicitPrognosticComponent(ImplicitPrognosticComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -1387,8 +1387,8 @@ class MyImplicitPrognostic(ImplicitPrognostic): def array_call(self, state, timestep): pass - instance = MyImplicitPrognostic() - assert not isinstance(instance, Prognostic) + instance = MyImplicitPrognosticComponent() + assert not isinstance(instance, PrognosticComponent) def test_ducktype_is_not_prognostic(self): class MyImplicitPrognostic(object): @@ -1403,10 +1403,10 @@ def array_call(self, state, timestep): pass instance = MyImplicitPrognostic() - assert not isinstance(instance, Prognostic) + assert not isinstance(instance, PrognosticComponent) def test_timedelta_is_passed(self): - prognostic = MockImplicitPrognostic({}, {}, {}, {}, {}) + prognostic = MockImplicitPrognosticComponent({}, {}, {}, {}, {}) tendencies, diagnostics = prognostic( {'time': timedelta(seconds=0)}, timedelta(seconds=5)) assert tendencies == {} @@ -1417,7 +1417,7 @@ def test_timedelta_is_passed(self): class DiagnosticTests(unittest.TestCase, InputTestBase, DiagnosticTestBase): - component_class = MockDiagnostic + component_class = MockDiagnosticComponent def call_component(self, component, state): return component(state) @@ -1426,7 +1426,7 @@ def get_component( self, input_properties=None, diagnostic_properties=None, diagnostic_output=None): - return MockDiagnostic( + return MockDiagnosticComponent( input_properties=input_properties or {}, diagnostic_properties=diagnostic_properties or {}, diagnostic_output=diagnostic_output or {}, @@ -1436,7 +1436,7 @@ def get_diagnostics(self, result): return result def test_cannot_use_bad_component(self): - component = BadMockDiagnostic() + component = BadMockDiagnosticComponent() with self.assertRaises(RuntimeError): self.call_component(component, {'time': timedelta(0)}) @@ -1450,27 +1450,27 @@ def array_call(self, state): pass instance = MyDiagnostic() - assert isinstance(instance, Diagnostic) + assert isinstance(instance, DiagnosticComponent) def test_two_components_are_not_instances_of_each_other(self): - class MyDiagnostic1(Diagnostic): + class MyDiagnosticComponent1(DiagnosticComponent): input_properties = {} diagnostic_properties = {} def array_call(self, state): pass - class MyDiagnostic2(Diagnostic): + class MyDiagnosticComponent2(DiagnosticComponent): input_properties = {} diagnostic_properties = {} def array_call(self, state): pass - diag1 = MyDiagnostic1() - diag2 = MyDiagnostic2() - assert not isinstance(diag1, MyDiagnostic2) - assert not isinstance(diag2, MyDiagnostic1) + diag1 = MyDiagnosticComponent1() + diag2 = MyDiagnosticComponent2() + assert not isinstance(diag1, MyDiagnosticComponent2) + assert not isinstance(diag2, MyDiagnosticComponent1) def test_ducktype_not_instance_of_subclass(self): class MyDiagnostic1(object): @@ -1481,7 +1481,7 @@ def __init__(self): def array_call(self, state): pass - class MyDiagnostic2(Diagnostic): + class MyDiagnosticComponent2(DiagnosticComponent): input_properties = {} diagnostic_properties = {} @@ -1489,7 +1489,7 @@ def array_call(self, state): pass diag1 = MyDiagnostic1() - assert not isinstance(diag1, MyDiagnostic2) + assert not isinstance(diag1, MyDiagnosticComponent2) def test_empty_diagnostic(self): diagnostic = self.component_class({}, {}, {}) @@ -1503,7 +1503,7 @@ def test_empty_diagnostic(self): class ImplicitTests(unittest.TestCase, InputTestBase, DiagnosticTestBase): - component_class = MockImplicit + component_class = MockStepper def call_component(self, component, state): return component(state, timedelta(seconds=1)) @@ -1512,7 +1512,7 @@ def get_component( self, input_properties=None, output_properties=None, diagnostic_properties=None, state_output=None, diagnostic_output=None): - return MockImplicit( + return MockStepper( input_properties=input_properties or {}, output_properties=output_properties or {}, diagnostic_properties=diagnostic_properties or {}, @@ -1528,7 +1528,7 @@ def test_raises_on_output_properties_of_wrong_type(self): self.get_component(output_properties=({},)) def test_cannot_use_bad_component(self): - component = BadMockImplicit() + component = BadMockStepper() with self.assertRaises(RuntimeError): self.call_component(component, {'time': timedelta(0)}) @@ -1545,7 +1545,7 @@ def array_call(self, state, timestep): pass instance = MyImplicit() - assert isinstance(instance, Implicit) + assert isinstance(instance, Stepper) def test_output_raises_when_units_incompatible_with_input(self): input_properties = { @@ -1561,7 +1561,7 @@ def test_output_raises_when_units_incompatible_with_input(self): ) def test_two_components_are_not_instances_of_each_other(self): - class MyImplicit1(Implicit): + class MyStepper1(Stepper): input_properties = {} diagnostic_properties = {} output_properties = {} @@ -1571,7 +1571,7 @@ class MyImplicit1(Implicit): def array_call(self, state): pass - class MyImplicit2(Implicit): + class MyStepper2(Stepper): input_properties = {} diagnostic_properties = {} output_properties = {} @@ -1581,10 +1581,10 @@ class MyImplicit2(Implicit): def array_call(self, state): pass - implicit1 = MyImplicit1() - implicit2 = MyImplicit2() - assert not isinstance(implicit1, MyImplicit2) - assert not isinstance(implicit2, MyImplicit1) + implicit1 = MyStepper1() + implicit2 = MyStepper2() + assert not isinstance(implicit1, MyStepper2) + assert not isinstance(implicit2, MyStepper1) def test_ducktype_not_instance_of_subclass(self): class MyImplicit1(object): @@ -1598,7 +1598,7 @@ def __init__(self): def array_call(self, state): pass - class MyImplicit2(Implicit): + class MyStepper2(Stepper): input_properties = {} diagnostic_properties = {} output_properties = {} @@ -1609,7 +1609,7 @@ def array_call(self, state): pass implicit1 = MyImplicit1() - assert not isinstance(implicit1, MyImplicit2) + assert not isinstance(implicit1, MyStepper2) def test_empty_implicit(self): implicit = self.component_class( @@ -1693,7 +1693,7 @@ def test_cannot_overlap_output_aliases(self): ) def test_timedelta_is_passed(self): - implicit = MockImplicit({}, {}, {}, {}, {}) + implicit = MockStepper({}, {}, {}, {}, {}) tendencies, diagnostics = implicit( {'time': timedelta(seconds=0)}, timedelta(seconds=5)) assert tendencies == {} @@ -1877,7 +1877,7 @@ def test_tendencies_in_diagnostics_no_tendency(self): output_properties = {} diagnostic_output = {} output_state = {} - implicit = MockImplicit( + implicit = MockStepper( input_properties, diagnostic_properties, output_properties, diagnostic_output, output_state, tendencies_in_diagnostics=True ) @@ -1901,17 +1901,17 @@ def test_tendencies_in_diagnostics_one_tendency(self): output_state = { 'output1': np.ones([10]) * 20., } - implicit = MockImplicit( + implicit = MockStepper( input_properties, diagnostic_properties, output_properties, diagnostic_output, output_state, tendencies_in_diagnostics=True, ) assert len(implicit.diagnostic_properties) == 1 - assert 'output1_tendency_from_mockimplicit' in implicit.diagnostic_properties.keys() - assert 'output1' in input_properties.keys(), 'Implicit needs original value to calculate tendency' + assert 'output1_tendency_from_MockStepper' in implicit.diagnostic_properties.keys() + assert 'output1' in input_properties.keys(), 'Stepper needs original value to calculate tendency' assert input_properties['output1']['dims'] == ['dim1'] assert input_properties['output1']['units'] == 'm' properties = implicit.diagnostic_properties[ - 'output1_tendency_from_mockimplicit'] + 'output1_tendency_from_MockStepper'] assert properties['dims'] == ['dim1'] assert properties['units'] == 'm s^-1' state = { @@ -1923,13 +1923,13 @@ def test_tendencies_in_diagnostics_one_tendency(self): ), } diagnostics, _ = implicit(state, timedelta(seconds=5)) - assert 'output1_tendency_from_mockimplicit' in diagnostics.keys() + assert 'output1_tendency_from_MockStepper' in diagnostics.keys() assert len( - diagnostics['output1_tendency_from_mockimplicit'].dims) == 1 - assert 'dim1' in diagnostics['output1_tendency_from_mockimplicit'].dims - assert diagnostics['output1_tendency_from_mockimplicit'].attrs['units'] == 'm s^-1' + diagnostics['output1_tendency_from_MockStepper'].dims) == 1 + assert 'dim1' in diagnostics['output1_tendency_from_MockStepper'].dims + assert diagnostics['output1_tendency_from_MockStepper'].attrs['units'] == 'm s^-1' assert np.all( - diagnostics['output1_tendency_from_mockimplicit'].values == 2.) + diagnostics['output1_tendency_from_MockStepper'].values == 2.) def test_tendencies_in_diagnostics_one_tendency_dims_from_input(self): input_properties = { @@ -1948,17 +1948,17 @@ def test_tendencies_in_diagnostics_one_tendency_dims_from_input(self): output_state = { 'output1': np.ones([10]) * 20., } - implicit = MockImplicit( + implicit = MockStepper( input_properties, diagnostic_properties, output_properties, diagnostic_output, output_state, tendencies_in_diagnostics=True, ) assert len(implicit.diagnostic_properties) == 1 - assert 'output1_tendency_from_mockimplicit' in implicit.diagnostic_properties.keys() - assert 'output1' in input_properties.keys(), 'Implicit needs original value to calculate tendency' + assert 'output1_tendency_from_MockStepper' in implicit.diagnostic_properties.keys() + assert 'output1' in input_properties.keys(), 'Stepper needs original value to calculate tendency' assert input_properties['output1']['dims'] == ['dim1'] assert input_properties['output1']['units'] == 'm' properties = implicit.diagnostic_properties[ - 'output1_tendency_from_mockimplicit'] + 'output1_tendency_from_MockStepper'] assert properties['dims'] == ['dim1'] assert properties['units'] == 'm s^-1' state = { @@ -1970,13 +1970,13 @@ def test_tendencies_in_diagnostics_one_tendency_dims_from_input(self): ), } diagnostics, _ = implicit(state, timedelta(seconds=5)) - assert 'output1_tendency_from_mockimplicit' in diagnostics.keys() + assert 'output1_tendency_from_MockStepper' in diagnostics.keys() assert len( - diagnostics['output1_tendency_from_mockimplicit'].dims) == 1 - assert 'dim1' in diagnostics['output1_tendency_from_mockimplicit'].dims - assert diagnostics['output1_tendency_from_mockimplicit'].attrs['units'] == 'm s^-1' + diagnostics['output1_tendency_from_MockStepper'].dims) == 1 + assert 'dim1' in diagnostics['output1_tendency_from_MockStepper'].dims + assert diagnostics['output1_tendency_from_MockStepper'].attrs['units'] == 'm s^-1' assert np.all( - diagnostics['output1_tendency_from_mockimplicit'].values == 2.) + diagnostics['output1_tendency_from_MockStepper'].values == 2.) def test_tendencies_in_diagnostics_one_tendency_mismatched_units(self): input_properties = { @@ -1997,7 +1997,7 @@ def test_tendencies_in_diagnostics_one_tendency_mismatched_units(self): 'output1': np.ones([10]) * 20., } with self.assertRaises(InvalidPropertyDictError): - implicit = MockImplicit( + implicit = MockStepper( input_properties, diagnostic_properties, output_properties, diagnostic_output, output_state, tendencies_in_diagnostics=True, ) @@ -2021,7 +2021,7 @@ def test_tendencies_in_diagnostics_one_tendency_mismatched_dims(self): 'output1': np.ones([10]) * 20., } with self.assertRaises(InvalidPropertyDictError): - implicit = MockImplicit( + implicit = MockStepper( input_properties, diagnostic_properties, output_properties, diagnostic_output, output_state, tendencies_in_diagnostics=True, ) @@ -2039,7 +2039,7 @@ def test_tendencies_in_diagnostics_one_tendency_with_component_name(self): output_state = { 'output1': np.ones([10]) * 7., } - implicit = MockImplicit( + implicit = MockStepper( input_properties, diagnostic_properties, output_properties, diagnostic_output, output_state, tendencies_in_diagnostics=True, name='component' diff --git a/tests/test_components.py b/tests/test_components.py index b8494b0..5bd5d33 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -1,13 +1,13 @@ import pytest from sympl import ( - ConstantPrognostic, ConstantDiagnostic, RelaxationPrognostic, DataArray, + ConstantPrognosticComponent, ConstantDiagnosticComponent, RelaxationPrognosticComponent, DataArray, timedelta, ) import numpy as np def test_constant_prognostic_empty_dicts(): - prog = ConstantPrognostic({}, {}) + prog = ConstantPrognosticComponent({}, {}) tendencies, diagnostics = prog({'time': timedelta(0)}) assert isinstance(tendencies, dict) assert isinstance(diagnostics, dict) @@ -18,7 +18,7 @@ def test_constant_prognostic_empty_dicts(): def test_constant_prognostic_cannot_modify_through_input_dict(): in_tendencies = {} in_diagnostics = {} - prog = ConstantPrognostic(in_tendencies, in_diagnostics) + prog = ConstantPrognosticComponent(in_tendencies, in_diagnostics) in_tendencies['a'] = 'b' in_diagnostics['c'] = 'd' tendencies, diagnostics = prog({'time': timedelta(0)}) @@ -27,7 +27,7 @@ def test_constant_prognostic_cannot_modify_through_input_dict(): def test_constant_prognostic_cannot_modify_through_output_dict(): - prog = ConstantPrognostic({}, {}) + prog = ConstantPrognosticComponent({}, {}) tendencies, diagnostics = prog({'time': timedelta(0)}) tendencies['a'] = 'b' diagnostics['c'] = 'd' @@ -49,7 +49,7 @@ def test_constant_prognostic_tendency_properties(): attrs={'units': 'degK/s'}, ) } - prog = ConstantPrognostic(tendencies) + prog = ConstantPrognosticComponent(tendencies) assert prog.tendency_properties == { 'tend1': { 'dims': ('dim1',), @@ -78,7 +78,7 @@ def test_constant_prognostic_diagnostic_properties(): attrs={'units': 'degK'}, ) } - prog = ConstantPrognostic(tendencies, diagnostics) + prog = ConstantPrognosticComponent(tendencies, diagnostics) assert prog.diagnostic_properties == { 'diag1': { 'dims': ('dim1',), @@ -94,7 +94,7 @@ def test_constant_prognostic_diagnostic_properties(): def test_constant_diagnostic_empty_dict(): - diag = ConstantDiagnostic({}) + diag = ConstantDiagnosticComponent({}) diagnostics = diag({'time': timedelta(0)}) assert isinstance(diagnostics, dict) assert len(diagnostics) == 0 @@ -102,7 +102,7 @@ def test_constant_diagnostic_empty_dict(): def test_constant_diagnostic_cannot_modify_through_input_dict(): in_diagnostics = {} - diag = ConstantDiagnostic(in_diagnostics) + diag = ConstantDiagnosticComponent(in_diagnostics) in_diagnostics['a'] = 'b' diagnostics = diag({'time': timedelta(0)}) assert isinstance(diagnostics, dict) @@ -110,7 +110,7 @@ def test_constant_diagnostic_cannot_modify_through_input_dict(): def test_constant_diagnostic_cannot_modify_through_output_dict(): - diag = ConstantDiagnostic({}) + diag = ConstantDiagnosticComponent({}) diagnostics = diag({'time': timedelta(0)}) diagnostics['c'] = 'd' diagnostics = diag({'time': timedelta(0)}) @@ -130,7 +130,7 @@ def test_constant_diagnostic_diagnostic_properties(): attrs={'units': 'degK'}, ) } - diagnostic = ConstantDiagnostic(diagnostics) + diagnostic = ConstantDiagnosticComponent(diagnostics) assert diagnostic.diagnostic_properties == { 'diag1': { 'dims': ('dim1',), @@ -145,7 +145,7 @@ def test_constant_diagnostic_diagnostic_properties(): def test_relaxation_prognostic_at_equilibrium(): - prognostic = RelaxationPrognostic('quantity', 'degK') + prognostic = RelaxationPrognosticComponent('quantity', 'degK') state = { 'time': timedelta(0), 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'degK'}), @@ -159,7 +159,7 @@ def test_relaxation_prognostic_at_equilibrium(): def test_relaxation_prognostic_with_change(): - prognostic = RelaxationPrognostic('quantity', 'degK') + prognostic = RelaxationPrognosticComponent('quantity', 'degK') state = { 'time': timedelta(0), 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'degK'}), @@ -173,7 +173,7 @@ def test_relaxation_prognostic_with_change(): def test_relaxation_prognostic_with_change_different_timescale_units(): - prognostic = RelaxationPrognostic('quantity', 'degK') + prognostic = RelaxationPrognosticComponent('quantity', 'degK') state = { 'time': timedelta(0), 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'degK'}), @@ -187,7 +187,7 @@ def test_relaxation_prognostic_with_change_different_timescale_units(): def test_relaxation_prognostic_with_change_different_equilibrium_units(): - prognostic = RelaxationPrognostic('quantity', 'm') + prognostic = RelaxationPrognosticComponent('quantity', 'm') state = { 'time': timedelta(0), 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'm'}), diff --git a/tests/test_composite.py b/tests/test_composite.py index 20ce596..e70a392 100644 --- a/tests/test_composite.py +++ b/tests/test_composite.py @@ -2,7 +2,7 @@ import unittest import mock from sympl import ( - Prognostic, Diagnostic, Monitor, PrognosticComposite, DiagnosticComposite, + PrognosticComponent, DiagnosticComponent, Monitor, PrognosticComponentComposite, DiagnosticComponentComposite, MonitorComposite, SharedKeyError, DataArray, InvalidPropertyDictError ) from sympl._core.units import units_are_compatible @@ -13,7 +13,7 @@ def same_list(list1, list2): [item in list2 for item in list1] + [item in list1 for item in list2])) -class MockPrognostic(Prognostic): +class MockPrognosticComponent(PrognosticComponent): input_properties = None diagnostic_properties = None @@ -29,7 +29,7 @@ def __init__( self._tendency_output = tendency_output self.times_called = 0 self.state_given = None - super(MockPrognostic, self).__init__(**kwargs) + super(MockPrognosticComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -37,7 +37,7 @@ def array_call(self, state): return self._tendency_output, self._diagnostic_output -class MockDiagnostic(Diagnostic): +class MockDiagnosticComponent(DiagnosticComponent): input_properties = None diagnostic_properties = None @@ -50,7 +50,7 @@ def __init__( self._diagnostic_output = diagnostic_output self.times_called = 0 self.state_given = None - super(MockDiagnostic, self).__init__(**kwargs) + super(MockDiagnosticComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -58,39 +58,39 @@ def array_call(self, state): return self._diagnostic_output -class MockEmptyPrognostic(Prognostic): +class MockEmptyPrognosticComponent(PrognosticComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} def __init__(self, **kwargs): - super(MockEmptyPrognostic, self).__init__(**kwargs) + super(MockEmptyPrognosticComponent, self).__init__(**kwargs) def array_call(self, state): return {}, {} -class MockEmptyPrognostic2(Prognostic): +class MockEmptyPrognosticComponent2(PrognosticComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} def __init__(self, **kwargs): - super(MockEmptyPrognostic2, self).__init__(**kwargs) + super(MockEmptyPrognosticComponent2, self).__init__(**kwargs) def array_call(self, state): return {}, {} -class MockEmptyDiagnostic(Diagnostic): +class MockEmptyDiagnosticComponent(DiagnosticComponent): input_properties = {} diagnostic_properties = {} def __init__(self, **kwargs): - super(MockEmptyDiagnostic, self).__init__(**kwargs) + super(MockEmptyDiagnosticComponent, self).__init__(**kwargs) def array_call(self, state): return {} @@ -103,7 +103,7 @@ def store(self, state): def test_empty_prognostic_composite(): - prognostic_composite = PrognosticComposite() + prognostic_composite = PrognosticComponentComposite() state = {'air_temperature': 273.15} tendencies, diagnostics = prognostic_composite(state) assert len(tendencies) == 0 @@ -112,10 +112,10 @@ def test_empty_prognostic_composite(): assert isinstance(diagnostics, dict) -@mock.patch.object(MockEmptyPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_prognostic_composite_calls_one_prognostic(mock_call): mock_call.return_value = ({'air_temperature': 0.5}, {'foo': 50.}) - prognostic_composite = PrognosticComposite(MockEmptyPrognostic()) + prognostic_composite = PrognosticComponentComposite(MockEmptyPrognosticComponent()) state = {'air_temperature': 273.15} tendencies, diagnostics = prognostic_composite(state) assert mock_call.called @@ -123,11 +123,11 @@ def test_prognostic_composite_calls_one_prognostic(mock_call): assert diagnostics == {'foo': 50.} -@mock.patch.object(MockEmptyPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_prognostic_composite_calls_two_prognostics(mock_call): mock_call.return_value = ({'air_temperature': 0.5}, {}) - prognostic_composite = PrognosticComposite( - MockEmptyPrognostic(), MockEmptyPrognostic()) + prognostic_composite = PrognosticComponentComposite( + MockEmptyPrognosticComponent(), MockEmptyPrognosticComponent()) state = {'air_temperature': 273.15} tendencies, diagnostics = prognostic_composite(state) assert mock_call.called @@ -137,17 +137,17 @@ def test_prognostic_composite_calls_two_prognostics(mock_call): def test_empty_diagnostic_composite(): - diagnostic_composite = DiagnosticComposite() + diagnostic_composite = DiagnosticComponentComposite() state = {'air_temperature': 273.15} diagnostics = diagnostic_composite(state) assert len(diagnostics) == 0 assert isinstance(diagnostics, dict) -@mock.patch.object(MockEmptyDiagnostic, '__call__') +@mock.patch.object(MockEmptyDiagnosticComponent, '__call__') def test_diagnostic_composite_calls_one_diagnostic(mock_call): mock_call.return_value = {'foo': 50.} - diagnostic_composite = DiagnosticComposite(MockEmptyDiagnostic()) + diagnostic_composite = DiagnosticComponentComposite(MockEmptyDiagnosticComponent()) state = {'air_temperature': 273.15} diagnostics = diagnostic_composite(state) assert mock_call.called @@ -182,7 +182,7 @@ def test_monitor_collection_calls_two_monitors(mock_store): def test_prognostic_composite_cannot_use_diagnostic(): try: - PrognosticComposite(MockEmptyDiagnostic()) + PrognosticComponentComposite(MockEmptyDiagnosticComponent()) except TypeError: pass except Exception as err: @@ -193,7 +193,7 @@ def test_prognostic_composite_cannot_use_diagnostic(): def test_diagnostic_composite_cannot_use_prognostic(): try: - DiagnosticComposite(MockEmptyPrognostic()) + DiagnosticComponentComposite(MockEmptyPrognosticComponent()) except TypeError: pass except Exception as err: @@ -202,11 +202,11 @@ def test_diagnostic_composite_cannot_use_prognostic(): raise AssertionError('TypeError should have been raised') -@mock.patch.object(MockEmptyDiagnostic, '__call__') +@mock.patch.object(MockEmptyDiagnosticComponent, '__call__') def test_diagnostic_composite_call(mock_call): mock_call.return_value = {'foo': 5.} state = {'bar': 10.} - diagnostics = DiagnosticComposite(MockEmptyDiagnostic()) + diagnostics = DiagnosticComponentComposite(MockEmptyDiagnosticComponent()) new_state = diagnostics(state) assert list(state.keys()) == ['bar'] assert state['bar'] == 10. @@ -214,16 +214,16 @@ def test_diagnostic_composite_call(mock_call): assert new_state['foo'] == 5. -@mock.patch.object(MockEmptyPrognostic, '__call__') -@mock.patch.object(MockEmptyPrognostic2, '__call__') +@mock.patch.object(MockEmptyPrognosticComponent, '__call__') +@mock.patch.object(MockEmptyPrognosticComponent2, '__call__') def test_prognostic_component_handles_units_when_combining(mock_call, mock2_call): mock_call.return_value = ({ 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})}, {}) mock2_call.return_value = ({ 'eastward_wind': DataArray(50., attrs={'units': 'cm/s'})}, {}) - prognostic1 = MockEmptyPrognostic() - prognostic2 = MockEmptyPrognostic2() - composite = PrognosticComposite(prognostic1, prognostic2) + prognostic1 = MockEmptyPrognosticComponent() + prognostic2 = MockEmptyPrognosticComponent2() + composite = PrognosticComponentComposite(prognostic1, prognostic2) tendencies, diagnostics = composite({}) assert tendencies['eastward_wind'].to_units('m/s').values.item() == 1.5 @@ -241,9 +241,9 @@ def test_diagnostic_composite_single_component_input(): } diagnostic_properties = {} diagnostic_output = {} - diagnostic = MockDiagnostic( + diagnostic = MockDiagnosticComponent( input_properties, diagnostic_properties, diagnostic_output) - composite = DiagnosticComposite(diagnostic) + composite = DiagnosticComponentComposite(diagnostic) assert composite.input_properties == input_properties assert composite.diagnostic_properties == diagnostic_properties @@ -261,9 +261,9 @@ def test_diagnostic_composite_single_component_diagnostic(): }, } diagnostic_output = {} - diagnostic = MockDiagnostic( + diagnostic = MockDiagnosticComponent( input_properties, diagnostic_properties, diagnostic_output) - composite = DiagnosticComposite(diagnostic) + composite = DiagnosticComponentComposite(diagnostic) assert composite.input_properties == input_properties assert composite.diagnostic_properties == diagnostic_properties @@ -272,9 +272,9 @@ def test_diagnostic_composite_single_empty_component(): input_properties = {} diagnostic_properties = {} diagnostic_output = {} - diagnostic = MockDiagnostic( + diagnostic = MockDiagnosticComponent( input_properties, diagnostic_properties, diagnostic_output) - composite = DiagnosticComposite(diagnostic) + composite = DiagnosticComponentComposite(diagnostic) assert composite.input_properties == input_properties assert composite.diagnostic_properties == diagnostic_properties @@ -301,9 +301,9 @@ def test_diagnostic_composite_single_full_component(): }, } diagnostic_output = {} - diagnostic = MockDiagnostic( + diagnostic = MockDiagnosticComponent( input_properties, diagnostic_properties, diagnostic_output) - composite = DiagnosticComposite(diagnostic) + composite = DiagnosticComponentComposite(diagnostic) assert composite.input_properties == input_properties assert composite.diagnostic_properties == diagnostic_properties @@ -321,9 +321,9 @@ def test_diagnostic_composite_single_component_no_dims_on_diagnostic(): }, } diagnostic_output = {} - diagnostic = MockDiagnostic( + diagnostic = MockDiagnosticComponent( input_properties, diagnostic_properties, diagnostic_output) - composite = DiagnosticComposite(diagnostic) + composite = DiagnosticComponentComposite(diagnostic) assert composite.input_properties == input_properties assert composite.diagnostic_properties == diagnostic_properties @@ -337,9 +337,9 @@ def test_diagnostic_composite_single_component_missing_dims_on_diagnostic(): } diagnostic_output = {} try: - diagnostic = MockDiagnostic( + diagnostic = MockDiagnosticComponent( input_properties, diagnostic_properties, diagnostic_output) - DiagnosticComposite(diagnostic) + DiagnosticComponentComposite(diagnostic) except InvalidPropertyDictError: pass else: @@ -347,7 +347,7 @@ def test_diagnostic_composite_single_component_missing_dims_on_diagnostic(): def test_diagnostic_composite_two_components_no_overlap(): - diagnostic1 = MockDiagnostic( + diagnostic1 = MockDiagnosticComponent( input_properties={ 'input1': { 'dims': ['dim1'], @@ -362,7 +362,7 @@ def test_diagnostic_composite_two_components_no_overlap(): }, diagnostic_output={} ) - diagnostic2 = MockDiagnostic( + diagnostic2 = MockDiagnosticComponent( input_properties={ 'input2': { 'dims': ['dim2'], @@ -377,7 +377,7 @@ def test_diagnostic_composite_two_components_no_overlap(): }, diagnostic_output={} ) - composite = DiagnosticComposite(diagnostic1, diagnostic2) + composite = DiagnosticComponentComposite(diagnostic1, diagnostic2) input_properties = { 'input1': { 'dims': ['dim1'], @@ -403,7 +403,7 @@ def test_diagnostic_composite_two_components_no_overlap(): def test_diagnostic_composite_two_components_overlap_input(): - diagnostic1 = MockDiagnostic( + diagnostic1 = MockDiagnosticComponent( input_properties={ 'input1': { 'dims': ['dim1'], @@ -422,7 +422,7 @@ def test_diagnostic_composite_two_components_overlap_input(): }, diagnostic_output={} ) - diagnostic2 = MockDiagnostic( + diagnostic2 = MockDiagnosticComponent( input_properties={ 'input1': { 'dims': ['dim1'], @@ -441,7 +441,7 @@ def test_diagnostic_composite_two_components_overlap_input(): }, diagnostic_output={} ) - composite = DiagnosticComposite(diagnostic1, diagnostic2) + composite = DiagnosticComponentComposite(diagnostic1, diagnostic2) input_properties = { 'input1': { 'dims': ['dim1'], @@ -467,7 +467,7 @@ def test_diagnostic_composite_two_components_overlap_input(): def test_diagnostic_composite_two_components_overlap_diagnostic(): - diagnostic1 = MockDiagnostic( + diagnostic1 = MockDiagnosticComponent( input_properties={}, diagnostic_properties={ 'diag1': { @@ -477,7 +477,7 @@ def test_diagnostic_composite_two_components_overlap_diagnostic(): }, diagnostic_output={} ) - diagnostic2 = MockDiagnostic( + diagnostic2 = MockDiagnosticComponent( input_properties={}, diagnostic_properties={ 'diag1': { @@ -488,7 +488,7 @@ def test_diagnostic_composite_two_components_overlap_diagnostic(): diagnostic_output={} ) try: - DiagnosticComposite(diagnostic1, diagnostic2) + DiagnosticComponentComposite(diagnostic1, diagnostic2) except SharedKeyError: pass else: @@ -496,7 +496,7 @@ def test_diagnostic_composite_two_components_overlap_diagnostic(): def test_diagnostic_composite_two_components_incompatible_input_dims(): - diagnostic1 = MockDiagnostic( + diagnostic1 = MockDiagnosticComponent( input_properties={ 'input1': { 'dims': ['dim1'], @@ -506,7 +506,7 @@ def test_diagnostic_composite_two_components_incompatible_input_dims(): diagnostic_properties={}, diagnostic_output={} ) - diagnostic2 = MockDiagnostic( + diagnostic2 = MockDiagnosticComponent( input_properties={ 'input1': { 'dims': ['dim2'], @@ -517,7 +517,7 @@ def test_diagnostic_composite_two_components_incompatible_input_dims(): diagnostic_output={} ) try: - composite = DiagnosticComposite(diagnostic1, diagnostic2) + composite = DiagnosticComponentComposite(diagnostic1, diagnostic2) except InvalidPropertyDictError: pass else: @@ -525,7 +525,7 @@ def test_diagnostic_composite_two_components_incompatible_input_dims(): def test_diagnostic_composite_two_components_incompatible_input_units(): - diagnostic1 = MockDiagnostic( + diagnostic1 = MockDiagnosticComponent( input_properties={ 'input1': { 'dims': ['dim1'], @@ -535,7 +535,7 @@ def test_diagnostic_composite_two_components_incompatible_input_units(): diagnostic_properties={}, diagnostic_output={} ) - diagnostic2 = MockDiagnostic( + diagnostic2 = MockDiagnosticComponent( input_properties={ 'input1': { 'dims': ['dim1'], @@ -546,7 +546,7 @@ def test_diagnostic_composite_two_components_incompatible_input_units(): diagnostic_output={} ) try: - DiagnosticComposite(diagnostic1, diagnostic2) + DiagnosticComponentComposite(diagnostic1, diagnostic2) except InvalidPropertyDictError: pass else: @@ -554,7 +554,7 @@ def test_diagnostic_composite_two_components_incompatible_input_units(): def test_prognostic_composite_single_input(): - prognostic = MockPrognostic( + prognostic = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1'], @@ -566,14 +566,14 @@ def test_prognostic_composite_single_input(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic) + composite = PrognosticComponentComposite(prognostic) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_prognostic_composite_single_diagnostic(): - prognostic = MockPrognostic( + prognostic = MockPrognosticComponent( input_properties={}, diagnostic_properties={ 'diag1': { @@ -585,14 +585,14 @@ def test_prognostic_composite_single_diagnostic(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic) + composite = PrognosticComponentComposite(prognostic) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_prognostic_composite_single_tendency(): - prognostic = MockPrognostic( + prognostic = MockPrognosticComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -604,14 +604,14 @@ def test_prognostic_composite_single_tendency(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic) + composite = PrognosticComponentComposite(prognostic) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_prognostic_composite_implicit_dims(): - prognostic = MockPrognostic( + prognostic = MockPrognosticComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -627,7 +627,7 @@ def test_prognostic_composite_implicit_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic) + composite = PrognosticComponentComposite(prognostic) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == { @@ -639,7 +639,7 @@ def test_prognostic_composite_implicit_dims(): def test_two_prognostic_composite_implicit_dims(): - prognostic = MockPrognostic( + prognostic = MockPrognosticComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -655,7 +655,7 @@ def test_two_prognostic_composite_implicit_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -671,7 +671,7 @@ def test_two_prognostic_composite_implicit_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic, prognostic2) + composite = PrognosticComponentComposite(prognostic, prognostic2) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == { @@ -683,7 +683,7 @@ def test_two_prognostic_composite_implicit_dims(): def test_prognostic_composite_explicit_dims(): - prognostic = MockPrognostic( + prognostic = MockPrognosticComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -700,14 +700,14 @@ def test_prognostic_composite_explicit_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic) + composite = PrognosticComponentComposite(prognostic) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_two_prognostic_composite_explicit_dims(): - prognostic = MockPrognostic( + prognostic = MockPrognosticComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -724,7 +724,7 @@ def test_two_prognostic_composite_explicit_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -741,14 +741,14 @@ def test_two_prognostic_composite_explicit_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic, prognostic2) + composite = PrognosticComponentComposite(prognostic, prognostic2) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_two_prognostic_composite_explicit_and_implicit_dims(): - prognostic = MockPrognostic( + prognostic = MockPrognosticComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -765,7 +765,7 @@ def test_two_prognostic_composite_explicit_and_implicit_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -781,14 +781,14 @@ def test_two_prognostic_composite_explicit_and_implicit_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic, prognostic2) + composite = PrognosticComponentComposite(prognostic, prognostic2) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_prognostic_composite_explicit_dims_not_in_input(): - prognostic = MockPrognostic( + prognostic = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -805,14 +805,14 @@ def test_prognostic_composite_explicit_dims_not_in_input(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic) + composite = PrognosticComponentComposite(prognostic) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_two_prognostic_composite_incompatible_dims(): - prognostic = MockPrognostic( + prognostic = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -833,7 +833,7 @@ def test_two_prognostic_composite_incompatible_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -855,7 +855,7 @@ def test_two_prognostic_composite_incompatible_dims(): tendency_output={}, ) try: - PrognosticComposite(prognostic, prognostic2) + PrognosticComponentComposite(prognostic, prognostic2) except InvalidPropertyDictError: pass else: @@ -863,7 +863,7 @@ def test_two_prognostic_composite_incompatible_dims(): def test_two_prognostic_composite_compatible_dims(): - prognostic = MockPrognostic( + prognostic = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -884,7 +884,7 @@ def test_two_prognostic_composite_compatible_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -905,14 +905,14 @@ def test_two_prognostic_composite_compatible_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic, prognostic2) + composite = PrognosticComponentComposite(prognostic, prognostic2) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_prognostic_composite_two_components_input(): - prognostic1 = MockPrognostic( + prognostic1 = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -928,7 +928,7 @@ def test_prognostic_composite_two_components_input(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -944,7 +944,7 @@ def test_prognostic_composite_two_components_input(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic1, prognostic2) + composite = PrognosticComponentComposite(prognostic1, prognostic2) input_properties = { 'input1': { 'dims': ['dims1', 'dims2'], @@ -967,7 +967,7 @@ def test_prognostic_composite_two_components_input(): def test_prognostic_composite_two_components_swapped_input_dims(): - prognostic1 = MockPrognostic( + prognostic1 = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -979,7 +979,7 @@ def test_prognostic_composite_two_components_swapped_input_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims2', 'dims1'], @@ -991,7 +991,7 @@ def test_prognostic_composite_two_components_swapped_input_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic1, prognostic2) + composite = PrognosticComponentComposite(prognostic1, prognostic2) diagnostic_properties = {} tendency_properties = {} assert (composite.input_properties == prognostic1.input_properties or @@ -1001,7 +1001,7 @@ def test_prognostic_composite_two_components_swapped_input_dims(): def test_prognostic_composite_two_components_incompatible_input_dims(): - prognostic1 = MockPrognostic( + prognostic1 = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -1013,7 +1013,7 @@ def test_prognostic_composite_two_components_incompatible_input_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims3'], @@ -1026,7 +1026,7 @@ def test_prognostic_composite_two_components_incompatible_input_dims(): tendency_output={}, ) try: - PrognosticComposite(prognostic1, prognostic2) + PrognosticComponentComposite(prognostic1, prognostic2) except InvalidPropertyDictError: pass else: @@ -1034,7 +1034,7 @@ def test_prognostic_composite_two_components_incompatible_input_dims(): def test_prognostic_composite_two_components_incompatible_input_units(): - prognostic1 = MockPrognostic( + prognostic1 = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -1046,7 +1046,7 @@ def test_prognostic_composite_two_components_incompatible_input_units(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -1059,7 +1059,7 @@ def test_prognostic_composite_two_components_incompatible_input_units(): tendency_output={}, ) try: - PrognosticComposite(prognostic1, prognostic2) + PrognosticComponentComposite(prognostic1, prognostic2) except InvalidPropertyDictError: pass else: @@ -1067,7 +1067,7 @@ def test_prognostic_composite_two_components_incompatible_input_units(): def test_prognostic_composite_two_components_compatible_input_units(): - prognostic1 = MockPrognostic( + prognostic1 = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -1079,7 +1079,7 @@ def test_prognostic_composite_two_components_compatible_input_units(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -1091,14 +1091,14 @@ def test_prognostic_composite_two_components_compatible_input_units(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic1, prognostic2) + composite = PrognosticComponentComposite(prognostic1, prognostic2) assert 'input1' in composite.input_properties.keys() assert composite.input_properties['input1']['dims'] == ['dims1', 'dims2'] assert units_are_compatible(composite.input_properties['input1']['units'], 'm') def test_prognostic_composite_two_components_tendency(): - prognostic1 = MockPrognostic( + prognostic1 = MockPrognosticComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1110,7 +1110,7 @@ def test_prognostic_composite_two_components_tendency(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1126,7 +1126,7 @@ def test_prognostic_composite_two_components_tendency(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic1, prognostic2) + composite = PrognosticComponentComposite(prognostic1, prognostic2) input_properties = {} diagnostic_properties = {} tendency_properties = { @@ -1145,7 +1145,7 @@ def test_prognostic_composite_two_components_tendency(): def test_prognostic_composite_two_components_tendency_incompatible_dims(): - prognostic1 = MockPrognostic( + prognostic1 = MockPrognosticComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1157,7 +1157,7 @@ def test_prognostic_composite_two_components_tendency_incompatible_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1174,7 +1174,7 @@ def test_prognostic_composite_two_components_tendency_incompatible_dims(): tendency_output={}, ) try: - PrognosticComposite(prognostic1, prognostic2) + PrognosticComponentComposite(prognostic1, prognostic2) except InvalidPropertyDictError: pass else: @@ -1182,7 +1182,7 @@ def test_prognostic_composite_two_components_tendency_incompatible_dims(): def test_prognostic_composite_two_components_tendency_incompatible_units(): - prognostic1 = MockPrognostic( + prognostic1 = MockPrognosticComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1194,7 +1194,7 @@ def test_prognostic_composite_two_components_tendency_incompatible_units(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1211,7 +1211,7 @@ def test_prognostic_composite_two_components_tendency_incompatible_units(): tendency_output={}, ) try: - PrognosticComposite(prognostic1, prognostic2) + PrognosticComponentComposite(prognostic1, prognostic2) except InvalidPropertyDictError: pass else: @@ -1219,7 +1219,7 @@ def test_prognostic_composite_two_components_tendency_incompatible_units(): def test_prognostic_composite_two_components_tendency_compatible_units(): - prognostic1 = MockPrognostic( + prognostic1 = MockPrognosticComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1231,7 +1231,7 @@ def test_prognostic_composite_two_components_tendency_compatible_units(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1243,14 +1243,14 @@ def test_prognostic_composite_two_components_tendency_compatible_units(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic1, prognostic2) + composite = PrognosticComponentComposite(prognostic1, prognostic2) assert 'tend1' in composite.tendency_properties.keys() assert composite.tendency_properties['tend1']['dims'] == ['dim1'] assert units_are_compatible(composite.tendency_properties['tend1']['units'], 'm/s') def test_prognostic_composite_two_components_diagnostic(): - prognostic1 = MockPrognostic( + prognostic1 = MockPrognosticComponent( input_properties={}, diagnostic_properties={ 'diag1': { @@ -1262,7 +1262,7 @@ def test_prognostic_composite_two_components_diagnostic(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={}, diagnostic_properties={ 'diag2': { @@ -1274,7 +1274,7 @@ def test_prognostic_composite_two_components_diagnostic(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComposite(prognostic1, prognostic2) + composite = PrognosticComponentComposite(prognostic1, prognostic2) input_properties = {} diagnostic_properties = { 'diag1': { @@ -1293,7 +1293,7 @@ def test_prognostic_composite_two_components_diagnostic(): def test_prognostic_composite_two_components_overlapping_diagnostic(): - prognostic1 = MockPrognostic( + prognostic1 = MockPrognosticComponent( input_properties={}, diagnostic_properties={ 'diag1': { @@ -1305,7 +1305,7 @@ def test_prognostic_composite_two_components_overlapping_diagnostic(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognostic( + prognostic2 = MockPrognosticComponent( input_properties={}, diagnostic_properties={ 'diag1': { @@ -1318,7 +1318,7 @@ def test_prognostic_composite_two_components_overlapping_diagnostic(): tendency_output={}, ) try: - PrognosticComposite(prognostic1, prognostic2) + PrognosticComponentComposite(prognostic1, prognostic2) except SharedKeyError: pass else: diff --git a/tests/test_time_differencing_wrapper.py b/tests/test_time_differencing_wrapper.py index 8e6873b..a7daaa7 100644 --- a/tests/test_time_differencing_wrapper.py +++ b/tests/test_time_differencing_wrapper.py @@ -1,14 +1,14 @@ from datetime import timedelta, datetime import unittest from sympl import ( - Prognostic, Implicit, Diagnostic, TimeDifferencingWrapper, DataArray + PrognosticComponent, Stepper, DiagnosticComponent, TimeDifferencingWrapper, DataArray ) import pytest from numpy.testing import assert_allclose from copy import deepcopy -class MockPrognostic(Prognostic): +class MockPrognosticComponent(PrognosticComponent): def __init__(self): self._num_updates = 0 @@ -18,7 +18,7 @@ def __call__(self, state): return {}, {'num_updates': self._num_updates} -class MockImplicit(Implicit): +class MockStepper(Stepper): input_properties = {} @@ -38,7 +38,7 @@ class MockImplicit(Implicit): def __init__(self): self._num_updates = 0 - super(MockImplicit, self).__init__() + super(MockStepper, self).__init__() def array_call(self, state, timestep): self._num_updates += 1 @@ -49,7 +49,7 @@ def array_call(self, state, timestep): ) -class MockImplicitThatExpects(Implicit): +class MockStepperThatExpects(Stepper): input_properties = {'expected_field': {}} output_properties = {'expected_field': {}} @@ -68,7 +68,7 @@ def __call__(self, state, timestep): return deepcopy(state), state -class MockPrognosticThatExpects(Prognostic): +class MockPrognosticComponentThatExpects(PrognosticComponent): input_properties = {'expected_field': {}} tendency_properties = {'expected_field': {}} @@ -87,7 +87,7 @@ def __call__(self, state): return deepcopy(state), state -class MockDiagnosticThatExpects(Diagnostic): +class MockDiagnosticComponentThatExpects(DiagnosticComponent): input_properties = {'expected_field': {}} diagnostic_properties = {'expected_field': {}} @@ -108,7 +108,7 @@ def __call__(self, state): class TimeDifferencingTests(unittest.TestCase): def setUp(self): - self.implicit = MockImplicit() + self.implicit = MockStepper() self.wrapped = TimeDifferencingWrapper(self.implicit) self.state = { 'time': timedelta(0), diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index 003b15c..9fdece4 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -1,8 +1,8 @@ import pytest import mock from sympl import ( - Prognostic, Leapfrog, AdamsBashforth, DataArray, SSPRungeKutta, timedelta, - InvalidPropertyDictError, ImplicitPrognostic) + PrognosticComponent, Leapfrog, AdamsBashforth, DataArray, SSPRungeKutta, timedelta, + InvalidPropertyDictError, ImplicitPrognosticComponent) from sympl._core.units import units_are_compatible import numpy as np import warnings @@ -13,7 +13,7 @@ def same_list(list1, list2): [item in list2 for item in list1] + [item in list1 for item in list2])) -class MockEmptyPrognostic(Prognostic): +class MockEmptyPrognosticComponent(PrognosticComponent): input_properties = {} tendency_properties = {} @@ -23,7 +23,7 @@ def array_call(self, state): return {}, {} -class MockPrognostic(Prognostic): +class MockPrognosticComponent(PrognosticComponent): input_properties = None diagnostic_properties = None @@ -39,7 +39,7 @@ def __init__( self._tendency_output = tendency_output self.times_called = 0 self.state_given = None - super(MockPrognostic, self).__init__(**kwargs) + super(MockPrognosticComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -47,7 +47,7 @@ def array_call(self, state): return self._tendency_output, self._diagnostic_output -class MockImplicitPrognostic(ImplicitPrognostic): +class MockImplicitPrognosticComponent(ImplicitPrognosticComponent): input_properties = None diagnostic_properties = None tendency_properties = None @@ -62,7 +62,7 @@ def __init__( self._tendency_output = tendency_output self.times_called = 0 self.state_given = None - super(MockImplicitPrognostic, self).__init__(**kwargs) + super(MockImplicitPrognosticComponent, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -72,7 +72,7 @@ def array_call(self, state, timestep): class PrognosticBase(object): - prognostic_class = MockPrognostic + prognostic_class = MockPrognosticComponent def test_given_tendency_not_modified_with_two_components(self): input_properties = {} @@ -408,7 +408,7 @@ def test_stepper_gives_diagnostic_tendency_quantity(self): class ImplicitPrognosticBase(PrognosticBase): - prognostic_class = MockImplicitPrognostic + prognostic_class = MockImplicitPrognosticComponent class TimesteppingBase(object): @@ -417,47 +417,47 @@ class TimesteppingBase(object): def test_unused_quantities_carried_over(self): state = {'time': timedelta(0), 'air_temperature': 273.} - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) timestep = timedelta(seconds=1.) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} def test_timestepper_reveals_prognostics(self): - prog1 = MockEmptyPrognostic() + prog1 = MockEmptyPrognosticComponent() prog1.input_properties = {'input1': {'dims': ['dim1'], 'units': 'm'}} time_stepper = self.timestepper_class(prog1) assert same_list(time_stepper.prognostic_list, (prog1,)) - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_float_no_change_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} assert diagnostics == {} - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_float_no_change_one_step_diagnostic(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': 0.}, {'foo': 'bar'}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} assert diagnostics == {'foo': 'bar'} - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_float_no_change_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} @@ -470,34 +470,34 @@ def test_float_no_change_three_steps(self, mock_prognostic_call): assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_float_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 1.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 274.} - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_float_one_step_with_units(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'eastward_wind': DataArray(0.02, attrs={'units': 'km/s^2'})}, {}) state = {'time': timedelta(0), 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} assert same_list(new_state.keys(), ['time', 'eastward_wind']) assert np.allclose(new_state['eastward_wind'].values, 21.) assert new_state['eastward_wind'].attrs['units'] == 'm/s' - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_float_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 1.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 274.} @@ -510,26 +510,26 @@ def test_float_three_steps(self, mock_prognostic_call): assert state == {'time': timedelta(0), 'air_temperature': 275.} assert new_state == {'time': timedelta(0), 'air_temperature': 276.} - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_array_no_change_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.zeros((3, 3))}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_array_no_change_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*0.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -548,26 +548,26 @@ def test_array_no_change_three_steps(self, mock_prognostic_call): assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_array_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*1.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*274.).all() - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_array_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*1.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -586,7 +586,7 @@ def test_array_three_steps(self, mock_prognostic_call): assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*276.).all() - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_dataarray_no_change_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': @@ -595,7 +595,7 @@ def test_dataarray_no_change_one_step(self, mock_prognostic_call): state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'].values == np.ones((3, 3))*273.).all() @@ -606,7 +606,7 @@ def test_dataarray_no_change_one_step(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_dataarray_no_change_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': @@ -615,7 +615,7 @@ def test_dataarray_no_change_three_steps(self, mock_prognostic_call): state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -642,7 +642,7 @@ def test_dataarray_no_change_three_steps(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_dataarray_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': @@ -651,7 +651,7 @@ def test_dataarray_one_step(self, mock_prognostic_call): state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -662,7 +662,7 @@ def test_dataarray_one_step(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_dataarray_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': @@ -671,7 +671,7 @@ def test_dataarray_three_steps(self, mock_prognostic_call): state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -698,13 +698,13 @@ def test_dataarray_three_steps(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' - @mock.patch.object(MockEmptyPrognostic, '__call__') + @mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_array_four_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3)) * 1.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3)) * 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognostic()) + time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3)) * 273.).all() @@ -823,13 +823,13 @@ def timestepper_class(self, *args, **kwargs): return AdamsBashforth(*args, **kwargs) -@mock.patch.object(MockEmptyPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_leapfrog_float_two_steps_filtered(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = Leapfrog(MockEmptyPrognostic(), asselin_strength=0.5, alpha=1.) + time_stepper = Leapfrog(MockEmptyPrognosticComponent(), asselin_strength=0.5, alpha=1.) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} @@ -841,12 +841,12 @@ def test_leapfrog_float_two_steps_filtered(mock_prognostic_call): assert new_state == {'time': timedelta(0), 'air_temperature': 277.} -@mock.patch.object(MockEmptyPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_leapfrog_requires_same_timestep(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} - time_stepper = Leapfrog([MockEmptyPrognostic()], asselin_strength=0.5) + time_stepper = Leapfrog([MockEmptyPrognosticComponent()], asselin_strength=0.5) diagnostics, state = time_stepper.__call__(state, timedelta(seconds=1.)) try: time_stepper.__call__(state, timedelta(seconds=2.)) @@ -858,12 +858,12 @@ def test_leapfrog_requires_same_timestep(mock_prognostic_call): raise AssertionError('Leapfrog must require timestep to be constant') -@mock.patch.object(MockEmptyPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_adams_bashforth_requires_same_timestep(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} - time_stepper = AdamsBashforth(MockEmptyPrognostic()) + time_stepper = AdamsBashforth(MockEmptyPrognosticComponent()) state = time_stepper.__call__(state, timedelta(seconds=1.)) try: time_stepper.__call__(state, timedelta(seconds=2.)) @@ -876,14 +876,14 @@ def test_adams_bashforth_requires_same_timestep(mock_prognostic_call): 'AdamsBashforth must require timestep to be constant') -@mock.patch.object(MockEmptyPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_leapfrog_array_two_steps_filtered(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*0.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = Leapfrog(MockEmptyPrognostic(), asselin_strength=0.5, alpha=1.) + time_stepper = Leapfrog(MockEmptyPrognosticComponent(), asselin_strength=0.5, alpha=1.) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -900,7 +900,7 @@ def test_leapfrog_array_two_steps_filtered(mock_prognostic_call): assert (new_state['air_temperature'] == np.ones((3, 3))*277.).all() -@mock.patch.object(MockEmptyPrognostic, '__call__') +@mock.patch.object(MockEmptyPrognosticComponent, '__call__') def test_leapfrog_array_two_steps_filtered_williams(mock_prognostic_call): """Test that the Asselin filter is being correctly applied with a Williams factor of alpha=0.5""" @@ -908,7 +908,7 @@ def test_leapfrog_array_two_steps_filtered_williams(mock_prognostic_call): {'air_temperature': np.ones((3, 3))*0.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = Leapfrog(MockEmptyPrognostic(), asselin_strength=0.5, alpha=0.5) + time_stepper = Leapfrog(MockEmptyPrognosticComponent(), asselin_strength=0.5, alpha=0.5) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() diff --git a/tests/test_tracers.py b/tests/test_tracers.py index 4e8e08f..f7c1448 100644 --- a/tests/test_tracers.py +++ b/tests/test_tracers.py @@ -1,6 +1,6 @@ from sympl._core.tracers import TracerPacker, clear_tracers, clear_packers from sympl import ( - Prognostic, Implicit, Diagnostic, ImplicitPrognostic, register_tracer, + PrognosticComponent, Stepper, DiagnosticComponent, ImplicitPrognosticComponent, register_tracer, get_tracer_unit_dict, units_are_compatible, DataArray ) import unittest @@ -9,7 +9,7 @@ from datetime import timedelta -class MockPrognostic(Prognostic): +class MockPrognosticComponent(PrognosticComponent): input_properties = None diagnostic_properties = None @@ -23,7 +23,7 @@ def __init__(self, **kwargs): self.tendency_output = {} self.times_called = 0 self.state_given = None - super(MockPrognostic, self).__init__(**kwargs) + super(MockPrognosticComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -31,7 +31,7 @@ def array_call(self, state): return self.tendency_output, self.diagnostic_output -class MockTracerPrognostic(Prognostic): +class MockTracerPrognosticComponent(PrognosticComponent): input_properties = None diagnostic_properties = None @@ -50,7 +50,7 @@ def __init__(self, **kwargs): self.diagnostic_output = {} self.times_called = 0 self.state_given = None - super(MockTracerPrognostic, self).__init__(**kwargs) + super(MockTracerPrognosticComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -61,7 +61,7 @@ def array_call(self, state): return return_state, self.diagnostic_output -class MockImplicitPrognostic(ImplicitPrognostic): +class MockImplicitPrognosticComponent(ImplicitPrognosticComponent): input_properties = None diagnostic_properties = None @@ -76,7 +76,7 @@ def __init__( self, **kwargs): self.times_called = 0 self.state_given = None self.timestep_given = None - super(MockImplicitPrognostic, self).__init__(**kwargs) + super(MockImplicitPrognosticComponent, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -85,7 +85,7 @@ def array_call(self, state, timestep): return self.tendency_output, self.diagnostic_output -class MockTracerImplicitPrognostic(ImplicitPrognostic): +class MockTracerImplicitPrognosticComponent(ImplicitPrognosticComponent): input_properties = None diagnostic_properties = None @@ -105,7 +105,7 @@ def __init__(self, **kwargs): self.times_called = 0 self.state_given = None self.timestep_given = None - super(MockTracerImplicitPrognostic, self).__init__(**kwargs) + super(MockTracerImplicitPrognosticComponent, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -117,7 +117,7 @@ def array_call(self, state, timestep): return return_state, self.diagnostic_output -class MockDiagnostic(Diagnostic): +class MockDiagnosticComponent(DiagnosticComponent): input_properties = None diagnostic_properties = None @@ -128,7 +128,7 @@ def __init__(self, **kwargs): self.diagnostic_output = {} self.times_called = 0 self.state_given = None - super(MockDiagnostic, self).__init__(**kwargs) + super(MockDiagnosticComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -136,7 +136,7 @@ def array_call(self, state): return self.diagnostic_output -class MockImplicit(Implicit): +class MockStepper(Stepper): input_properties = None diagnostic_properties = None @@ -151,7 +151,7 @@ def __init__(self, **kwargs): self.times_called = 0 self.state_given = None self.timestep_given = None - super(MockImplicit, self).__init__(**kwargs) + super(MockStepper, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -160,7 +160,7 @@ def array_call(self, state, timestep): return self.diagnostic_output, self.state_output -class MockTracerImplicit(Implicit): +class MockTracerStepper(Stepper): input_properties = None diagnostic_properties = None @@ -181,7 +181,7 @@ def __init__(self, **kwargs): self.times_called = 0 self.state_given = None self.timestep_given = None - super(MockTracerImplicit, self).__init__(**kwargs) + super(MockTracerStepper, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -375,7 +375,7 @@ def test_unpacks_three_tracers_in_order_registered(self): class PrognosticTracerPackerTests(TracerPackerBase, unittest.TestCase): def setUp(self): - self.component = MockPrognostic() + self.component = MockPrognosticComponent() super(PrognosticTracerPackerTests, self).setUp() def tearDown(self): @@ -408,7 +408,7 @@ def test_packs_updates_tendency_properties_after_init(self): class ImplicitPrognosticTracerPackerTests(PrognosticTracerPackerTests): def setUp(self): - self.component = MockImplicitPrognostic() + self.component = MockImplicitPrognosticComponent() super(ImplicitPrognosticTracerPackerTests, self).setUp() def tearDown(self): @@ -419,7 +419,7 @@ def tearDown(self): class ImplicitTracerPackerTests(TracerPackerBase, unittest.TestCase): def setUp(self): - self.component = MockImplicit() + self.component = MockStepper() super(ImplicitTracerPackerTests, self).setUp() def tearDown(self): @@ -452,7 +452,7 @@ def test_packs_updates_output_properties_after_init(self): class DiagnosticTracerPackerTests(unittest.TestCase): def test_raises_on_diagnostic_init(self): - diagnostic = MockDiagnostic() + diagnostic = MockDiagnosticComponent() with self.assertRaises(TypeError): TracerPacker(diagnostic, ['tracer', '*']) @@ -702,7 +702,7 @@ class PrognosticTracerComponentTests(TracerComponentBase, unittest.TestCase): def setUp(self): super(PrognosticTracerComponentTests, self).setUp() - self.component = MockTracerPrognostic() + self.component = MockTracerPrognosticComponent() def tearDown(self): super(PrognosticTracerComponentTests, self).tearDown() @@ -716,7 +716,7 @@ class ImplicitPrognosticTracerComponentTests(TracerComponentBase, unittest.TestC def setUp(self): super(ImplicitPrognosticTracerComponentTests, self).setUp() - self.component = MockTracerImplicitPrognostic() + self.component = MockTracerImplicitPrognosticComponent() def tearDown(self): super(ImplicitPrognosticTracerComponentTests, self).tearDown() @@ -730,7 +730,7 @@ class ImplicitTracerComponentTests(TracerComponentBase, unittest.TestCase): def setUp(self): super(ImplicitTracerComponentTests, self).setUp() - self.component = MockTracerImplicit() + self.component = MockTracerStepper() def tearDown(self): super(ImplicitTracerComponentTests, self).tearDown() diff --git a/tests/test_util.py b/tests/test_util.py index a2c692e..1d48922 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -2,8 +2,8 @@ import numpy as np import pytest from sympl import ( - Prognostic, ensure_no_shared_keys, SharedKeyError, DataArray, - Implicit, Diagnostic, + PrognosticComponent, ensure_no_shared_keys, SharedKeyError, DataArray, + Stepper, DiagnosticComponent, InvalidPropertyDictError) from sympl._core.util import update_dict_by_adding_another, combine_dims, get_component_aliases @@ -64,7 +64,7 @@ def test_update_dict_by_adding_another_adds_shared_arrays_reversed(): assert len(dict2.keys()) == 2 -class DummyPrognostic(Prognostic): +class DummyPrognosticComponent(PrognosticComponent): input_properties = {'temperature': {'alias': 'T'}} diagnostic_properties = {'pressure': {'alias': 'P'}} tendency_properties = {'temperature': {}} diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index 9cef4b8..6d57122 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -1,14 +1,14 @@ from datetime import timedelta, datetime import unittest from sympl import ( - Prognostic, Implicit, Diagnostic, UpdateFrequencyWrapper, ScalingWrapper, - TimeDifferencingWrapper, DataArray, ImplicitPrognostic + PrognosticComponent, Stepper, DiagnosticComponent, UpdateFrequencyWrapper, ScalingWrapper, + TimeDifferencingWrapper, DataArray, ImplicitPrognosticComponent ) import pytest import numpy as np -class MockPrognostic(Prognostic): +class MockPrognosticComponent(PrognosticComponent): input_properties = None diagnostic_properties = None @@ -24,7 +24,7 @@ def __init__( self.tendency_output = tendency_output self.times_called = 0 self.state_given = None - super(MockPrognostic, self).__init__(**kwargs) + super(MockPrognosticComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -32,7 +32,7 @@ def array_call(self, state): return self.tendency_output, self.diagnostic_output -class MockImplicitPrognostic(ImplicitPrognostic): +class MockImplicitPrognosticComponent(ImplicitPrognosticComponent): input_properties = None diagnostic_properties = None @@ -49,7 +49,7 @@ def __init__( self.times_called = 0 self.state_given = None self.timestep_given = None - super(MockImplicitPrognostic, self).__init__(**kwargs) + super(MockImplicitPrognosticComponent, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -58,7 +58,7 @@ def array_call(self, state, timestep): return self.tendency_output, self.diagnostic_output -class MockDiagnostic(Diagnostic): +class MockDiagnosticComponent(DiagnosticComponent): input_properties = None diagnostic_properties = None @@ -71,7 +71,7 @@ def __init__( self.diagnostic_output = diagnostic_output self.times_called = 0 self.state_given = None - super(MockDiagnostic, self).__init__(**kwargs) + super(MockDiagnosticComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -79,7 +79,7 @@ def array_call(self, state): return self.diagnostic_output -class MockImplicit(Implicit): +class MockStepper(Stepper): input_properties = None diagnostic_properties = None @@ -97,7 +97,7 @@ def __init__( self.times_called = 0 self.state_given = None self.timestep_given = None - super(MockImplicit, self).__init__(**kwargs) + super(MockStepper, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -106,7 +106,7 @@ def array_call(self, state, timestep): return self.diagnostic_output, self.state_output -class MockEmptyPrognostic(MockPrognostic): +class MockEmptyPrognostic(MockPrognosticComponent): def __init__(self, **kwargs): super(MockEmptyPrognostic, self).__init__( @@ -119,7 +119,7 @@ def __init__(self, **kwargs): ) -class MockEmptyImplicitPrognostic(MockImplicitPrognostic): +class MockEmptyImplicitPrognostic(MockImplicitPrognosticComponent): def __init__(self, **kwargs): super(MockEmptyImplicitPrognostic, self).__init__( input_properties={}, @@ -131,7 +131,7 @@ def __init__(self, **kwargs): ) -class MockEmptyDiagnostic(MockDiagnostic): +class MockEmptyDiagnostic(MockDiagnosticComponent): def __init__(self, **kwargs): super(MockEmptyDiagnostic, self).__init__( @@ -142,7 +142,7 @@ def __init__(self, **kwargs): ) -class MockEmptyImplicit(MockImplicit): +class MockEmptyImplicit(MockStepper): def __init__(self, **kwargs): super(MockEmptyImplicit, self).__init__( @@ -210,7 +210,7 @@ def test_set_update_frequency_does_not_update_when_less(self): class PrognosticUpdateFrequencyTests(unittest.TestCase, UpdateFrequencyBase): - component_type = Prognostic + component_type = PrognosticComponent def get_component(self): return MockEmptyPrognostic() @@ -221,7 +221,7 @@ def call_component(self, component, state): class ImplicitPrognosticUpdateFrequencyTests(unittest.TestCase, UpdateFrequencyBase): - component_type = ImplicitPrognostic + component_type = ImplicitPrognosticComponent def get_component(self): return MockEmptyImplicitPrognostic() @@ -232,7 +232,7 @@ def call_component(self, component, state): class ImplicitUpdateFrequencyTests(unittest.TestCase, UpdateFrequencyBase): - component_type = Implicit + component_type = Stepper def get_component(self): return MockEmptyImplicit() @@ -243,7 +243,7 @@ def call_component(self, component, state): class DiagnosticUpdateFrequencyTests(unittest.TestCase, UpdateFrequencyBase): - component_type = Diagnostic + component_type = DiagnosticComponent def get_component(self): return MockEmptyDiagnostic() @@ -655,7 +655,7 @@ def test_tendency_no_scaling_when_input_scaled(self): class DiagnosticScalingTests( unittest.TestCase, ScalingInputMixin, ScalingDiagnosticMixin): - component_type = Diagnostic + component_type = DiagnosticComponent def setUp(self): self.input_properties = {} @@ -663,7 +663,7 @@ def setUp(self): self.diagnostic_output = {} def get_component(self): - return MockDiagnostic( + return MockDiagnosticComponent( self.input_properties, self.diagnostic_properties, self.diagnostic_output @@ -679,7 +679,7 @@ def call_component(self, component, state): class PrognosticScalingTests( unittest.TestCase, ScalingInputMixin, ScalingDiagnosticMixin, ScalingTendencyMixin): - component_type = Prognostic + component_type = PrognosticComponent def setUp(self): self.input_properties = {} @@ -689,7 +689,7 @@ def setUp(self): self.tendency_output = {} def get_component(self): - return MockPrognostic( + return MockPrognosticComponent( self.input_properties, self.diagnostic_properties, self.tendency_properties, @@ -711,7 +711,7 @@ class ImplicitPrognosticScalingTests( unittest.TestCase, ScalingInputMixin, ScalingDiagnosticMixin, ScalingTendencyMixin): - component_type = ImplicitPrognostic + component_type = ImplicitPrognosticComponent def setUp(self): self.input_properties = {} @@ -721,7 +721,7 @@ def setUp(self): self.tendency_output = {} def get_component(self): - return MockImplicitPrognostic( + return MockImplicitPrognosticComponent( self.input_properties, self.diagnostic_properties, self.tendency_properties, @@ -743,7 +743,7 @@ class ImplicitScalingTests( unittest.TestCase, ScalingInputMixin, ScalingDiagnosticMixin, ScalingOutputMixin): - component_type = Implicit + component_type = Stepper def setUp(self): self.input_properties = {} @@ -753,7 +753,7 @@ def setUp(self): self.output_state = {} def get_component(self): - return MockImplicit( + return MockStepper( self.input_properties, self.diagnostic_properties, self.output_properties, From e18ce8984e280329e101c2f43eb5abb884242a1c Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 25 Jul 2018 17:12:29 -0700 Subject: [PATCH 83/98] Renamed PrognosticComponent to TendencyComponent --- HISTORY.rst | 18 +-- docs/composites.rst | 6 +- docs/computation.rst | 40 +++--- docs/memory_management.rst | 6 +- docs/overview.rst | 4 +- docs/quickstart.rst | 10 +- docs/timestepping.rst | 10 +- docs/writing_components.rst | 24 ++-- sympl/__init__.py | 12 +- sympl/_components/__init__.py | 4 +- sympl/_components/basic.py | 18 +-- sympl/_components/timesteppers.py | 6 +- sympl/_core/base_components.py | 22 ++-- sympl/_core/composite.py | 26 ++-- sympl/_core/prognosticstepper.py | 41 ++++-- sympl/_core/tracers.py | 2 +- sympl/_core/util.py | 4 +- sympl/_core/wrappers.py | 24 ++-- tests/test_base_components.py | 72 +++++------ tests/test_components.py | 20 +-- tests/test_composite.py | 160 ++++++++++++------------ tests/test_time_differencing_wrapper.py | 6 +- tests/test_timestepping.py | 102 +++++++-------- tests/test_tracers.py | 26 ++-- tests/test_util.py | 4 +- tests/test_wrapper.py | 28 ++--- 26 files changed, 355 insertions(+), 340 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ce2ee73..e2dc0ef 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,7 +5,7 @@ What's New Latest ------ -* Stepper, DiagnosticComponent, ImplicitPrognosticComponent, and PrognosticComponent base classes were +* Stepper, DiagnosticComponent, ImplicitTendencyComponent, and TendencyComponent base classes were modified to include functionality that was previously in ScalingWrapper, UpdateFrequencyWrapper, and TendencyInDiagnosticsWrapper. The functionality of TendencyInDiagnosticsWrapper is now to be used in Stepper and PrognosticStepper objects. @@ -13,7 +13,7 @@ Latest composited. * TimeSteppers now have a prognostic_list attribute which contains the prognostics used to calculate tendencies. -* TimeSteppers from sympl can now handle ImplicitPrognosticComponent components. +* TimeSteppers from sympl can now handle ImplicitTendencyComponent components. * Added a check for netcdftime having the required objects, to fall back on not using netcdftime when those are missing. This is because most objects are missing in older versions of netcdftime (that come packaged with netCDF4) (closes #23). @@ -31,7 +31,7 @@ Latest * Added a priority order of property types for determining which aliases are returned by get_component_aliases. * Fixed a bug where PrognosticStepper objects would modify the arrays passed to them by - PrognosticComponent objects, leading to unexpected value changes. + TendencyComponent objects, leading to unexpected value changes. * Fixed a bug where constants were missing from the string returned by get_constants_string, particularly any new constants (issue #27) * Fixed a bug in NetCDFMonitor which led to some aliases being skipped. @@ -55,7 +55,7 @@ Breaking changes ~~~~~~~~~~~~~~~~ * Implicit, Timestepper, Prognostic, ImplicitPrognostic, and Diagnostic objects have been renamed to - PrognosticStepper, Stepper, PrognosticComponent, ImplicitPrognosticComponent, + PrognosticStepper, Stepper, TendencyComponent, ImplicitTendencyComponent, and DiagnosticComponent. These changes are also reflected in subclass names. * inputs, outputs, diagnostics, and tendencies are no longer attributes of components. In order to get these, you should use e.g. input_properties.keys() @@ -74,13 +74,13 @@ Breaking changes have been removed. The functionality of these wrappers has been moved to the component base types as methods and initialization options. * 'time' now must be present in the model state dictionary. This is strictly required - for calls to DiagnosticComponent, PrognosticComponent, ImplicitPrognosticComponent, and Stepper components, + for calls to DiagnosticComponent, TendencyComponent, ImplicitTendencyComponent, and Stepper components, and may be strictly required in other ways in the future * Removed everything to do with directional wildcards. Currently '*' is the only wildcard dimension. 'x', 'y', and 'z' refer to their own names only. * Removed the combine_dimensions function, which wasn't used anywhere and no longer has much purpose without directional wildcards -* RelaxationPrognosticComponent no longer allows caching of equilibrium values or +* RelaxationTendencyComponent no longer allows caching of equilibrium values or timescale. They must be provided through the input state. This is to ensure proper conversion of dimensions and units. * Removed ComponentTestBase from package. All of its tests except for output @@ -93,7 +93,7 @@ Breaking changes used instead. If present, `dims` from input properties will be used as default. * Components will now raise an exception when __call__ of the component base - class (e.g. Stepper, PrognosticComponent, etc.) if the __init__ method of the base + class (e.g. Stepper, TendencyComponent, etc.) if the __init__ method of the base class has not been called, telling the user that the component __init__ method should make a call to the superclass init. @@ -135,10 +135,10 @@ v0.3.0 restore_data_arrays_with_properties * corrected heat capacity of snow and ice to be floats instead of ints * Added get_constant function as the way to retrieve constants -* Added ImplicitPrognosticComponent as a new component type. It is like a PrognosticComponent, +* Added ImplicitTendencyComponent as a new component type. It is like a TendencyComponent, but its call signature also requires that a timestep be given. * Added TimeDifferencingWrapper, which turns an Stepper into an - ImplicitPrognosticComponent by applying first-order time differencing. + ImplicitTendencyComponent by applying first-order time differencing. * Added set_condensible_name as a way of changing what condensible aliases (for example, density_of_solid_phase) refer to. Default is 'water'. * Moved wrappers to their own file (out from util.py). diff --git a/docs/composites.rst b/docs/composites.rst index 5723806..a157d67 100644 --- a/docs/composites.rst +++ b/docs/composites.rst @@ -4,7 +4,7 @@ Composites There are a set of objects in Sympl that wrap multiple components into a single object so they can be called as if they were one component. There is one each -for :py:class:`~sympl.PrognosticComponent`, :py:class:`~sympl.DiagnosticComponent`, and +for :py:class:`~sympl.TendencyComponent`, :py:class:`~sympl.DiagnosticComponent`, and :py:class:`~sympl.Monitor`. These can be used to simplify code, so that the way you call a list of components is the same as the way you would call a single component. For example, *instead* of writing: @@ -35,7 +35,7 @@ You could write: .. code-block:: python - prognostic_composite = PrognosticComponentComposite([ + prognostic_composite = TendencyComponentComposite([ MyPrognostic(), MyOtherPrognostic(), YetAnotherPrognostic(), @@ -54,7 +54,7 @@ overwritten). You can get similar simplifications for API Reference ------------- -.. autoclass:: sympl.PrognosticComponentComposite +.. autoclass:: sympl.TendencyComponentComposite :members: :special-members: :exclude-members: __weakref__,__metaclass__ diff --git a/docs/computation.rst b/docs/computation.rst index e8f464e..f28f036 100644 --- a/docs/computation.rst +++ b/docs/computation.rst @@ -2,17 +2,17 @@ Component Types =============== -In Sympl, computation is mainly performed using :py:class:`~sympl.PrognosticComponent`, +In Sympl, computation is mainly performed using :py:class:`~sympl.TendencyComponent`, :py:class:`~sympl.DiagnosticComponent`, and :py:class:`~sympl.Stepper` objects. Each of these types, once initialized, can be passed in a current model state. -:py:class:`~sympl.PrognosticComponent` objects use the state to return tendencies and +:py:class:`~sympl.TendencyComponent` objects use the state to return tendencies and diagnostics at the current time. :py:class:`~sympl.DiagnosticComponent` objects return only diagnostics from the current time. :py:class:`~sympl.Stepper` objects will take in a timestep along with the state, and then return the next state as well as modifying the current state to include more diagnostics (it is similar to a :py:class:`~sympl.PrognosticStepper` in how it is called). -In specific cases, it may be necessary to use a :py:class:`~sympl.ImplicitPrognosticComponent` +In specific cases, it may be necessary to use a :py:class:`~sympl.ImplicitTendencyComponent` object, which is discussed at the end of this section. These classes themselves (listed in the previous paragraph) are not ones you @@ -27,15 +27,15 @@ for their inputs and outputs, which are described in the section Details on the internals of components and how to write them are in the section on :ref:`Writing Components`. -PrognosticComponent +TendencyComponent ---------- -As stated above, :py:class:`~sympl.PrognosticComponent` objects use the state to return +As stated above, :py:class:`~sympl.TendencyComponent` objects use the state to return tendencies and diagnostics at the current time. In a full model, the tendencies are used by a time stepping scheme (in Sympl, a :py:class:`~sympl.PrognosticStepper`) to determine the values of quantities at the next time. -You can call a :py:class:`~sympl.PrognosticComponent` directly to get diagnostics and +You can call a :py:class:`~sympl.TendencyComponent` directly to get diagnostics and tendencies like so: .. code-block:: python @@ -44,25 +44,25 @@ tendencies like so: diagnostics, tendencies = radiation(state) ``diagnostics`` and ``tendencies`` in this case will both be dictionaries, -similar to ``state``. Even if the :py:class:`~sympl.PrognosticComponent` being called +similar to ``state``. Even if the :py:class:`~sympl.TendencyComponent` being called does not compute any diagnostics, it will still return an empty diagnostics dictionary. -Usually, you will call a PrognosticComponent object through a +Usually, you will call a TendencyComponent object through a :py:class:`~sympl.PrognosticStepper` that uses it to determine values at the next timestep. -.. autoclass:: sympl.PrognosticComponent +.. autoclass:: sympl.TendencyComponent :members: :special-members: :exclude-members: __weakref__,__metaclass__ -.. autoclass:: sympl.ConstantPrognosticComponent +.. autoclass:: sympl.ConstantTendencyComponent :members: :special-members: :exclude-members: __weakref__,__metaclass__ -.. autoclass:: sympl.RelaxationPrognosticComponent +.. autoclass:: sympl.RelaxationTendencyComponent :members: :special-members: :exclude-members: __weakref__,__metaclass__ @@ -140,7 +140,7 @@ there are a number of attributes with names like ``input_properties`` for components. These attributes give a fairly complete description of the inputs and outputs of the component. -You can access them like this (for an example :py:class:`~sympl.PrognosticComponent` +You can access them like this (for an example :py:class:`~sympl.TendencyComponent` class ``RRTMRadiation``): .. code-block:: python @@ -282,22 +282,22 @@ equal to: that the object will output ``cloud_fraction`` in its diagnostics on the same grid as ``air_temperature``, in dimensionless units. -ImplicitPrognosticComponent +ImplicitTendencyComponent ------------------ .. warning:: This component type should be avoided unless you know you need it, for reasons discussed in this section. In addition to the component types described above, computation may be performed by a -:py:class:`~sympl.ImplicitPrognosticComponent`. This class should be avoided unless you +:py:class:`~sympl.ImplicitTendencyComponent`. This class should be avoided unless you know what you are doing, but it may be necessary in certain cases. An -:py:class:`~sympl.ImplicitPrognosticComponent`, like a :py:class:`~sympl.PrognosticComponent`, +:py:class:`~sympl.ImplicitTendencyComponent`, like a :py:class:`~sympl.TendencyComponent`, calculates tendencies, but it does so using both the model state and a timestep. Certain components, like ones handling advection using a spectral method, may need to derive tendencies from an :py:class:`~sympl.Stepper` object by -representing it using an :py:class:`~sympl.ImplicitPrognosticComponent`. +representing it using an :py:class:`~sympl.ImplicitTendencyComponent`. -The reason to avoid using an :py:class:`~sympl.ImplicitPrognosticComponent` is that if +The reason to avoid using an :py:class:`~sympl.ImplicitTendencyComponent` is that if a component requires a timestep, it is making internal assumptions about how you are timestepping. For example, it may use the timestep to ensure that all supersaturated water is condensed by the end of the timestep using an assumption @@ -306,7 +306,7 @@ which does not obey those assumptions, you may get unintended behavior, such as some supersaturated water remaining, or too much water being condensed. For this reason, the :py:class:`~sympl.PrognosticStepper` objects included in Sympl -do not wrap :py:class:`~sympl.ImplicitPrognosticComponent` components. If you would like +do not wrap :py:class:`~sympl.ImplicitTendencyComponent` components. If you would like to use this type of component, and know what you are doing, it is pretty easy to write your own :py:class:`~sympl.PrognosticStepper` to do so (you can base the code off of the code in Sympl), or the model you are using might already have @@ -314,13 +314,13 @@ components to do this for you. If you are wrapping a parameterization and notice that it needs a timestep to compute its tendencies, that is likely *not* a good reason to write an -:py:class:`~sympl.ImplicitPrognosticComponent`. If at all possible you should modify the +:py:class:`~sympl.ImplicitTendencyComponent`. If at all possible you should modify the code to compute the value at the next timestep, and write an :py:class:`~sympl.Stepper` component. You are welcome to reach out to the developers of Sympl if you would like advice on your specific situation! We're always excited about new wrapped components. -.. autoclass:: sympl.ImplicitPrognosticComponent +.. autoclass:: sympl.ImplicitTendencyComponent :members: :special-members: :exclude-members: __weakref__,__metaclass__ diff --git a/docs/memory_management.rst b/docs/memory_management.rst index f337276..6598a80 100644 --- a/docs/memory_management.rst +++ b/docs/memory_management.rst @@ -10,15 +10,15 @@ Arrays If possible, you should try to be aware of when there are two code references to the same in-memory array. This can help avoid some common bugs. Let's start -with an example. Say you create a ConstantPrognosticComponent object like so:: +with an example. Say you create a ConstantTendencyComponent object like so:: >>> import numpy as np - >>> from sympl import ConstantPrognosticComponent, DataArray + >>> from sympl import ConstantTendencyComponent, DataArray >>> array = DataArray( np.ones((5, 5, 10)), dims=('lon', 'lat', 'lev'), attrs={'units': 'K/s'}) >>> tendencies = {'air_temperature': array} - >>> prognostic = ConstantPrognosticComponent(tendencies) + >>> prognostic = ConstantTendencyComponent(tendencies) This is all fine so far. But it's important to know that now ``array`` is the same array stored inside ``prognostic``:: diff --git a/docs/overview.rst b/docs/overview.rst index 6cbf3e3..7a35996 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -77,12 +77,12 @@ The state dictionary is evolved by :py:class:`~sympl.PrognosticStepper` and :py:class:`~sympl.Stepper` objects. These types of objects take in the state and a timedelta object that indicates the time step, and return the next model state. :py:class:`~sympl.PrognosticStepper` objects do this by wrapping -:py:class:`~sympl.PrognosticComponent` objects, which calculate tendencies using the +:py:class:`~sympl.TendencyComponent` objects, which calculate tendencies using the state dictionary. We should note that the meaning of "Stepper" in Sympl is slightly different than its traditional definition. Here an "Stepper" object is one that calculates the new state directly from the current state, or any object that requires the timestep to calculate the new state, while -"PrognosticComponent" objects are ones that calculate tendencies without using the +"TendencyComponent" objects are ones that calculate tendencies without using the timestep. If a :py:class:`~sympl.PrognosticStepper` or :py:class:`~sympl.Stepper` object needs to use multiple time steps in its calculation, it does so by storing states it was previously given until they are no longer needed. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 58a5da6..8d19d19 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -138,8 +138,8 @@ Those are the "components": implicit_dynamics = ImplicitDynamics() :py:class:`~sympl.AdamsBashforth` is a :py:class:`~sympl.PrognosticStepper`, which is -created with a set of :py:class:`~sympl.PrognosticComponent` components. -The :py:class:`~sympl.PrognosticComponent` components we have here are ``Radiation``, +created with a set of :py:class:`~sympl.TendencyComponent` components. +The :py:class:`~sympl.TendencyComponent` components we have here are ``Radiation``, ``BoundaryLayer``, and ``DeepConvection``. Each of these carries information about what it takes as inputs and provides as outputs, and can be called with a model state to return tendencies for a set of quantities. The @@ -147,16 +147,16 @@ state to return tendencies for a set of quantities. The forward in time. The :py:class:`~sympl.UpdateFrequencyWrapper` applied to the ``Radiation`` object -is an object that acts like a :py:class:`~sympl.PrognosticComponent` but only computes +is an object that acts like a :py:class:`~sympl.TendencyComponent` but only computes its output if at least a certain amount of model time has passed since the last time the output was computed. Otherwise, it returns the last computed output. This is commonly used in atmospheric models to avoid doing radiation calculations (which are very expensive) every timestep, but it can be applied -to any PrognosticComponent. +to any TendencyComponent. The :py:class:`ImplicitDynamics` class is a :py:class:`~sympl.Stepper` object, which steps the model state forward in time in the same way that a :py:class:`~sympl.PrognosticStepper` -would, but doesn't use :py:class:`~sympl.PrognosticComponent` objects in doing so. +would, but doesn't use :py:class:`~sympl.TendencyComponent` objects in doing so. The Main Loop ------------- diff --git a/docs/timestepping.rst b/docs/timestepping.rst index ecf66ba..70f140f 100644 --- a/docs/timestepping.rst +++ b/docs/timestepping.rst @@ -2,8 +2,8 @@ Timestepping ============ :py:class:`~sympl.PrognosticStepper` objects use time derivatives from -:py:class:`~sympl.PrognosticComponent` objects to step a model state forward in time. -They are initialized using any number of :py:class:`~sympl.PrognosticComponent` objects. +:py:class:`~sympl.TendencyComponent` objects to step a model state forward in time. +They are initialized using any number of :py:class:`~sympl.TendencyComponent` objects. .. code-block:: python @@ -46,13 +46,13 @@ For that reason, :py:class:`~sympl.PrognosticStepper` objects do not update There are also :py:class:`~sympl.Stepper` objects which evolve the state forward in time -without the use of PrognosticComponent objects. These function exactly the same as a +without the use of TendencyComponent objects. These function exactly the same as a :py:class:`~sympl.PrognosticStepper` once they are created, but do not accept -:py:class:`~sympl.PrognosticComponent` objects when you create them. One example might +:py:class:`~sympl.TendencyComponent` objects when you create them. One example might be a component that condenses all supersaturated moisture over some time period. :py:class:`~sympl.Stepper` objects are generally used for parameterizations that work by determining the target model state in some way, or involve -limiters, and cannot be represented as a :py:class:`~sympl.PrognosticComponent`. +limiters, and cannot be represented as a :py:class:`~sympl.TendencyComponent`. .. autoclass:: sympl.PrognosticStepper :members: diff --git a/docs/writing_components.rst b/docs/writing_components.rst index 234ca64..d93f4df 100644 --- a/docs/writing_components.rst +++ b/docs/writing_components.rst @@ -13,17 +13,17 @@ to talk about the parts of their code. Writing an Example ------------------ -Let's start with a PrognosticComponent component which relaxes temperature towards some +Let's start with a TendencyComponent component which relaxes temperature towards some target temperature. We'll go over the sections of this example step-by-step below. .. code-block:: python from sympl import ( - PrognosticComponent, get_numpy_arrays_with_properties, + TendencyComponent, get_numpy_arrays_with_properties, restore_data_arrays_with_properties) - class TemperatureRelaxation(PrognosticComponent): + class TemperatureRelaxation(TendencyComponent): input_properties = { 'air_temperature': { @@ -68,7 +68,7 @@ so that it can be found right away by anyone reading your code. .. code-block:: python from sympl import ( - PrognosticComponent, get_numpy_arrays_with_properties, + TendencyComponent, get_numpy_arrays_with_properties, restore_data_arrays_with_properties) Define an Object @@ -78,13 +78,13 @@ Once these are imported, there's this line: .. code-block:: python - class TemperatureRelaxation(PrognosticComponent): + class TemperatureRelaxation(TendencyComponent): This is the syntax for defining an object in Python. ``TemperatureRelaxation`` -will be the name of the new object. The :py:class:`~sympl.PrognosticComponent` +will be the name of the new object. The :py:class:`~sympl.TendencyComponent` in parentheses is telling Python that ``TemperatureRelaxation`` is a *subclass* of -:py:class:`~sympl.PrognosticComponent`. This tells Sympl that it can expect your object -to behave like a :py:class:`~sympl.PrognosticComponent`. +:py:class:`~sympl.TendencyComponent`. This tells Sympl that it can expect your object +to behave like a :py:class:`~sympl.TendencyComponent`. Define Attributes ***************** @@ -293,7 +293,7 @@ Using Tracers Sympl's base components have some features to automatically create tracer arrays for use by dynamical components. If an :py:class:`~sympl.Stepper`, -:py:class:`~sympl.PrognosticComponent`, or :py:class:`~sympl.ImplicitPrognosticComponent` +:py:class:`~sympl.TendencyComponent`, or :py:class:`~sympl.ImplicitTendencyComponent` component specifies ``uses_tracers = True`` and sets ``tracer_dims``, this feature is enabled. @@ -314,8 +314,8 @@ Once this feature is enabled, the ``state`` passed to ``array_call`` on the component will include a quantity called "tracers" with the dimensions specified by ``tracer_dims``. It will also be required that these tracers are used in the output. For a :py:class:`~sympl.Stepper` component, "tracers" -must be present in the output state, and for a :py:class:`~sympl.PrognosticComponent` or -:py:class:`~sympl.ImplicitPrognosticComponent` component "tracers" must be present in +must be present in the output state, and for a :py:class:`~sympl.TendencyComponent` or +:py:class:`~sympl.ImplicitTendencyComponent` component "tracers" must be present in the tendencies, with the same dimensions as the input "tracers". On these latter two components, you should also specify a @@ -326,7 +326,7 @@ units of ``g m^-3 s^-1``. This value is set as "s" (or seconds) by default. .. code-block:: python - class MyDynamicalCore(PrognosticComponent): + class MyDynamicalCore(TendencyComponent): uses_tracers = True tracer_dims = ['tracer', '*', 'mid_levels'] diff --git a/sympl/__init__.py b/sympl/__init__.py index 26638c5..203a4d3 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- from ._core.base_components import ( - PrognosticComponent, DiagnosticComponent, Stepper, Monitor, ImplicitPrognosticComponent + TendencyComponent, DiagnosticComponent, Stepper, Monitor, ImplicitTendencyComponent ) -from ._core.composite import PrognosticComponentComposite, DiagnosticComponentComposite, \ +from ._core.composite import TendencyComponentComposite, DiagnosticComponentComposite, \ MonitorComposite from ._core.prognosticstepper import PrognosticStepper from ._components.timesteppers import AdamsBashforth, Leapfrog, SSPRungeKutta @@ -28,15 +28,15 @@ initialize_numpy_arrays_with_properties) from ._components import ( PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, - ConstantPrognosticComponent, ConstantDiagnosticComponent, RelaxationPrognosticComponent, + ConstantTendencyComponent, ConstantDiagnosticComponent, RelaxationTendencyComponent, TimeDifferencingWrapper) from ._core.wrappers import UpdateFrequencyWrapper, ScalingWrapper from ._core.time import datetime, timedelta __version__ = '0.3.2' __all__ = ( - PrognosticComponent, DiagnosticComponent, Stepper, Monitor, PrognosticComponentComposite, - DiagnosticComponentComposite, MonitorComposite, ImplicitPrognosticComponent, + TendencyComponent, DiagnosticComponent, Stepper, Monitor, TendencyComponentComposite, + DiagnosticComponentComposite, MonitorComposite, ImplicitTendencyComponent, PrognosticStepper, Leapfrog, AdamsBashforth, SSPRungeKutta, InvalidStateError, SharedKeyError, DependencyError, InvalidPropertyDictError, ComponentExtraOutputError, @@ -53,7 +53,7 @@ initialize_numpy_arrays_with_properties, get_component_aliases, combine_component_properties, PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, - ConstantPrognosticComponent, ConstantDiagnosticComponent, RelaxationPrognosticComponent, + ConstantTendencyComponent, ConstantDiagnosticComponent, RelaxationTendencyComponent, UpdateFrequencyWrapper, ScalingWrapper, datetime, timedelta ) diff --git a/sympl/_components/__init__.py b/sympl/_components/__init__.py index 931548c..ba29856 100644 --- a/sympl/_components/__init__.py +++ b/sympl/_components/__init__.py @@ -1,11 +1,11 @@ from .netcdf import NetCDFMonitor, RestartMonitor from .plot import PlotFunctionMonitor from .basic import ( - ConstantPrognosticComponent, ConstantDiagnosticComponent, RelaxationPrognosticComponent, + ConstantTendencyComponent, ConstantDiagnosticComponent, RelaxationTendencyComponent, TimeDifferencingWrapper) __all__ = ( PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, - ConstantPrognosticComponent, ConstantDiagnosticComponent, RelaxationPrognosticComponent, + ConstantTendencyComponent, ConstantDiagnosticComponent, RelaxationTendencyComponent, TimeDifferencingWrapper) diff --git a/sympl/_components/basic.py b/sympl/_components/basic.py index a797c59..4f35e4f 100644 --- a/sympl/_components/basic.py +++ b/sympl/_components/basic.py @@ -1,9 +1,9 @@ from .._core.array import DataArray -from .._core.base_components import ImplicitPrognosticComponent, PrognosticComponent, DiagnosticComponent +from .._core.base_components import ImplicitTendencyComponent, TendencyComponent, DiagnosticComponent from .._core.units import unit_registry as ureg -class ConstantPrognosticComponent(PrognosticComponent): +class ConstantTendencyComponent(TendencyComponent): """ Prescribes constant tendencies provided at initialization. @@ -83,11 +83,11 @@ def __init__(self, tendencies, diagnostics=None, **kwargs): tendencies : dict A dictionary whose keys are strings indicating state quantities and values are the time derivative of those - quantities in units/second to be returned by this PrognosticComponent. + quantities in units/second to be returned by this TendencyComponent. diagnostics : dict, optional A dictionary whose keys are strings indicating state quantities and values are the value of those quantities - to be returned by this PrognosticComponent. By default an empty dictionary + to be returned by this TendencyComponent. By default an empty dictionary is used. input_scale_factors : dict, optional A (possibly empty) dictionary whose keys are quantity names and @@ -118,7 +118,7 @@ def __init__(self, tendencies, diagnostics=None, **kwargs): self.__diagnostics = diagnostics.copy() else: self.__diagnostics = {} - super(ConstantPrognosticComponent, self).__init__(**kwargs) + super(ConstantTendencyComponent, self).__init__(**kwargs) def array_call(self, state): tendencies = {} @@ -210,7 +210,7 @@ def array_call(self, state): return return_state -class RelaxationPrognosticComponent(PrognosticComponent): +class RelaxationTendencyComponent(TendencyComponent): r""" Applies Newtonian relaxation to a single quantity. @@ -319,7 +319,7 @@ def __init__(self, quantity_name, units, **kwargs): """ self._quantity_name = quantity_name self._units = units - super(RelaxationPrognosticComponent, self).__init__(**kwargs) + super(RelaxationTendencyComponent, self).__init__(**kwargs) def array_call(self, state): """ @@ -359,9 +359,9 @@ def array_call(self, state): return tendencies, {} -class TimeDifferencingWrapper(ImplicitPrognosticComponent): +class TimeDifferencingWrapper(ImplicitTendencyComponent): """ - Wraps an Stepper object and turns it into an ImplicitPrognosticComponent by applying + Wraps an Stepper object and turns it into an ImplicitTendencyComponent by applying simple first-order time differencing to determine tendencies. Example diff --git a/sympl/_components/timesteppers.py b/sympl/_components/timesteppers.py index 440f258..5c11561 100644 --- a/sympl/_components/timesteppers.py +++ b/sympl/_components/timesteppers.py @@ -16,7 +16,7 @@ def __init__(self, *args, **kwargs): Args ---- - *args : PrognosticComponent or ImplicitPrognosticComponent + *args : TendencyComponent or ImplicitTendencyComponent Objects to call for tendencies when doing time stepping. stages: int, optional Number of stages to use. Should be 2 or 3. Default is 3. @@ -80,7 +80,7 @@ def __init__(self, *args, **kwargs): Args ---- - *args : PrognosticComponent or ImplicitPrognosticComponent + *args : TendencyComponent or ImplicitTendencyComponent Objects to call for tendencies when doing time stepping. order : int, optional The order of accuracy to use. Must be between @@ -190,7 +190,7 @@ def __init__(self, *args, **kwargs): Args ---- - *args : PrognosticComponent or ImplicitPrognosticComponent + *args : TendencyComponent or ImplicitTendencyComponent Objects to call for tendencies when doing time stepping. asselin_strength : float, optional The filter parameter used to determine the strength diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index bb110e9..b81d935 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -26,11 +26,11 @@ def apply_scale_factors(array_state, scale_factors): def is_component_class(cls): - return any(issubclass(cls, cls2) for cls2 in (Stepper, PrognosticComponent, ImplicitPrognosticComponent, DiagnosticComponent)) + return any(issubclass(cls, cls2) for cls2 in (Stepper, TendencyComponent, ImplicitTendencyComponent, DiagnosticComponent)) def is_component_base_class(cls): - return cls in (Stepper, PrognosticComponent, ImplicitPrognosticComponent, DiagnosticComponent) + return cls in (Stepper, TendencyComponent, ImplicitTendencyComponent, DiagnosticComponent) def get_kwarg_defaults(func): @@ -668,7 +668,7 @@ def array_call(self, state, timestep): @add_metaclass(ComponentMeta) -class PrognosticComponent(object): +class TendencyComponent(object): """ Attributes ---------- @@ -711,7 +711,7 @@ def diagnostic_properties(self): def __str__(self): return ( - 'instance of {}(PrognosticComponent)\n' + 'instance of {}(TendencyComponent)\n' ' inputs: {}\n' ' tendencies: {}\n' ' diagnostics: {}'.format( @@ -766,7 +766,7 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): self._tracer_packer = TracerPacker( self, self.tracer_dims, prepend_tracers=prepend_tracers) self.__initialized = True - super(PrognosticComponent, self).__init__() + super(TendencyComponent, self).__init__() @property def tendencies_in_diagnostics(self): @@ -831,7 +831,7 @@ def __call__(self, state): KeyError If a required quantity is missing from the state. InvalidStateError - If state is not a valid input for the PrognosticComponent instance. + If state is not a valid input for the TendencyComponent instance. """ self._check_self_is_initialized() self._input_checker.check_inputs(state) @@ -892,7 +892,7 @@ def array_call(self, state): @add_metaclass(ComponentMeta) -class ImplicitPrognosticComponent(object): +class ImplicitTendencyComponent(object): """ Attributes ---------- @@ -935,7 +935,7 @@ def diagnostic_properties(self): def __str__(self): return ( - 'instance of {}(PrognosticComponent)\n' + 'instance of {}(TendencyComponent)\n' ' inputs: {}\n' ' tendencies: {}\n' ' diagnostics: {}'.format( @@ -987,7 +987,7 @@ def __init__(self, tendencies_in_diagnostics=False, name=None): self._tracer_packer = TracerPacker( self, self.tracer_dims, prepend_tracers=prepend_tracers) self.__initialized = True - super(ImplicitPrognosticComponent, self).__init__() + super(ImplicitTendencyComponent, self).__init__() @property def tendencies_in_diagnostics(self): @@ -1054,7 +1054,7 @@ def __call__(self, state, timestep): KeyError If a required quantity is missing from the state. InvalidStateError - If state is not a valid input for the PrognosticComponent instance. + If state is not a valid input for the TendencyComponent instance. """ self._check_self_is_initialized() self._input_checker.check_inputs(state) @@ -1205,7 +1205,7 @@ def __call__(self, state): KeyError If a required quantity is missing from the state. InvalidStateError - If state is not a valid input for the PrognosticComponent instance. + If state is not a valid input for the TendencyComponent instance. """ self._check_self_is_initialized() self._input_checker.check_inputs(state) diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 6eefb1e..080a5b3 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -1,4 +1,4 @@ -from .base_components import PrognosticComponent, DiagnosticComponent, Monitor, ImplicitPrognosticComponent +from .base_components import TendencyComponent, DiagnosticComponent, Monitor, ImplicitTendencyComponent from .util import ( update_dict_by_adding_another, ensure_no_shared_keys, combine_component_properties) @@ -99,11 +99,11 @@ def ensure_components_have_class(components, component_class): component_class, type(component))) -class PrognosticComponentComposite( +class TendencyComponentComposite( ComponentComposite, InputPropertiesCompositeMixin, - DiagnosticPropertiesCompositeMixin, PrognosticComponent): + DiagnosticPropertiesCompositeMixin, TendencyComponent): - component_class = PrognosticComponent + component_class = TendencyComponent @property def tendency_properties(self): @@ -126,7 +126,7 @@ def __init__(self, *args): output quantity, and their dimensions or units are incompatible with one another. """ - super(PrognosticComponentComposite, self).__init__(*args) + super(TendencyComponentComposite, self).__init__(*args) self.input_properties self.tendency_properties self.diagnostic_properties @@ -156,7 +156,7 @@ def __call__(self, state): KeyError If a required quantity is missing from the state. InvalidStateError - If state is not a valid input for a PrognosticComponent instance. + If state is not a valid input for a TendencyComponent instance. """ return_tendencies = {} return_diagnostics = {} @@ -170,11 +170,11 @@ def array_call(self, state): raise NotImplementedError() -class ImplicitPrognosticComponentComposite( +class ImplicitTendencyComponentComposite( ComponentComposite, InputPropertiesCompositeMixin, - DiagnosticPropertiesCompositeMixin, ImplicitPrognosticComponent): + DiagnosticPropertiesCompositeMixin, ImplicitTendencyComponent): - component_class = (PrognosticComponent, ImplicitPrognosticComponent) + component_class = (TendencyComponent, ImplicitTendencyComponent) @property def tendency_properties(self): @@ -197,7 +197,7 @@ def __init__(self, *args): output quantity, and their dimensions or units are incompatible with one another. """ - super(ImplicitPrognosticComponentComposite, self).__init__(*args) + super(ImplicitTendencyComponentComposite, self).__init__(*args) self.input_properties self.tendency_properties self.diagnostic_properties @@ -227,14 +227,14 @@ def __call__(self, state, timestep): KeyError If a required quantity is missing from the state. InvalidStateError - If state is not a valid input for a PrognosticComponent instance. + If state is not a valid input for a TendencyComponent instance. """ return_tendencies = {} return_diagnostics = {} for prognostic in self.component_list: - if isinstance(prognostic, ImplicitPrognosticComponent): + if isinstance(prognostic, ImplicitTendencyComponent): tendencies, diagnostics = prognostic(state, timestep) - elif isinstance(prognostic, PrognosticComponent): + elif isinstance(prognostic, TendencyComponent): tendencies, diagnostics = prognostic(state) update_dict_by_adding_another(return_tendencies, tendencies) return_diagnostics.update(diagnostics) diff --git a/sympl/_core/prognosticstepper.py b/sympl/_core/prognosticstepper.py index f81db45..1dab6ca 100644 --- a/sympl/_core/prognosticstepper.py +++ b/sympl/_core/prognosticstepper.py @@ -1,17 +1,18 @@ import abc -from .composite import ImplicitPrognosticComponentComposite +from .composite import ImplicitTendencyComponentComposite from .time import timedelta from .util import combine_component_properties, combine_properties from .units import clean_units from .state import copy_untouched_quantities -from .base_components import ImplicitPrognosticComponent, Stepper +from .base_components import ImplicitTendencyComponent, Stepper +from .exceptions import InvalidPropertyDictError import warnings class PrognosticStepper(Stepper): """An object which integrates model state forward in time. - It uses PrognosticComponent and DiagnosticComponent objects to update the current model state + It uses TendencyComponent and DiagnosticComponent objects to update the current model state with diagnostics, and to return the model state at the next timestep. Attributes @@ -26,11 +27,11 @@ class PrognosticStepper(Stepper): for the new state are returned when the object is called, and values are dictionaries which indicate 'dims' and 'units'. - prognostic : ImplicitPrognosticComponentComposite - A composite of the PrognosticComponent and ImplicitPrognostic objects used by + prognostic : ImplicitTendencyComponentComposite + A composite of the TendencyComponent and ImplicitPrognostic objects used by the PrognosticStepper. - prognostic_list: list of PrognosticComponent and ImplicitPrognosticComponent - A list of PrognosticComponent objects called by the PrognosticStepper. These should + prognostic_list: list of TendencyComponent and ImplicitPrognosticComponent + A list of TendencyComponent objects called by the PrognosticStepper. These should be referenced when determining what inputs are necessary for the PrognosticStepper. tendencies_in_diagnostics : bool @@ -94,7 +95,7 @@ def _tendency_properties(self): def __str__(self): return ( 'instance of {}(PrognosticStepper)\n' - ' PrognosticComponent components: {}'.format(self.prognostic_list) + ' TendencyComponent components: {}'.format(self.prognostic_list) ) def __repr__(self): @@ -119,7 +120,7 @@ def __init__(self, *args, **kwargs): Parameters ---------- - *args : PrognosticComponent or ImplicitPrognosticComponent + *args : TendencyComponent or ImplicitTendencyComponent Objects to call for tendencies when doing time stepping. tendencies_in_diagnostics : bool, optional A boolean indicating whether this object will put tendencies of @@ -135,13 +136,20 @@ def __init__(self, *args, **kwargs): 'than a list, and will not accept lists in a later version.', DeprecationWarning) args = args[0] - if any(isinstance(a, ImplicitPrognosticComponent) for a in args): + if any(isinstance(a, ImplicitTendencyComponent) for a in args): warnings.warn( - 'Using an ImplicitPrognosticComponent in sympl PrognosticStepper objects may ' + 'Using an ImplicitTendencyComponent in sympl PrognosticStepper objects may ' 'lead to scientifically invalid results. Make sure the component ' 'follows the same numerical assumptions as the PrognosticStepper used.') - self.prognostic = ImplicitPrognosticComponentComposite(*args) + self.prognostic = ImplicitTendencyComponentComposite(*args) super(PrognosticStepper, self).__init__(**kwargs) + for name in self.prognostic.tendency_properties.keys(): + if name not in self.output_properties.keys(): + raise InvalidPropertyDictError( + 'Prognostic object has tendency output for {} but ' + 'PrognosticStepper containing it does not have it in ' + 'output_properties.'.format(name)) + self.__initialized = True @property def prognostic_list(self): @@ -173,6 +181,13 @@ def __call__(self, state, timestep): new_state : dict The model state at the next timestep. """ + if not self.__initialized: + raise AssertionError( + 'PrognosticStepper component has not had its base class ' + '__init__ called, likely due to a missing call to ' + 'super(ClassName, self).__init__(*args, **kwargs) in its ' + '__init__ method.' + ) diagnostics, new_state = self._call(state, timestep) copy_untouched_quantities(state, new_state) if self.tendencies_in_diagnostics: @@ -188,7 +203,7 @@ def _insert_tendencies_to_diagnostics( tendency_name = self._get_tendency_name(name) if tendency_name in diagnostics.keys(): raise RuntimeError( - 'A PrognosticComponent has output tendencies as a diagnostic and has' + 'A TendencyComponent has output tendencies as a diagnostic and has' ' caused a name clash when trying to do so from this ' 'PrognosticStepper ({}). You must disable ' 'tendencies_in_diagnostics for this PrognosticStepper.'.format( diff --git a/sympl/_core/tracers.py b/sympl/_core/tracers.py index 51a28b8..8c1805f 100644 --- a/sympl/_core/tracers.py +++ b/sympl/_core/tracers.py @@ -84,7 +84,7 @@ def __init__(self, component, tracer_dims, prepend_tracers=None): else: raise TypeError( 'Expected a component object subclassing type Stepper, ' - 'ImplicitPrognosticComponent, or PrognosticComponent but received component of ' + 'ImplicitTendencyComponent, or TendencyComponent but received component of ' 'type {}'.format(component.__class__.__name__)) for name, units in self._prepend_tracers: if name not in _tracer_unit_dict.keys(): diff --git a/sympl/_core/util.py b/sympl/_core/util.py index 460f47a..5bf6f2f 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -420,8 +420,8 @@ def combine_properties(property_list, input_properties=None): def get_component_aliases(*args): """ - Returns aliases for variables in the properties of Components (PrognosticComponent, - DiagnosticComponent, Stepper, and ImplicitPrognosticComponent objects). + Returns aliases for variables in the properties of Components (TendencyComponent, + DiagnosticComponent, Stepper, and ImplicitTendencyComponent objects). If multiple aliases are present for the same variable, the following properties have priority in descending order: input, output, diagnostic, diff --git a/sympl/_core/wrappers.py b/sympl/_core/wrappers.py index 57c850c..5ff57a5 100644 --- a/sympl/_core/wrappers.py +++ b/sympl/_core/wrappers.py @@ -1,5 +1,5 @@ from .._core.base_components import ( - PrognosticComponent, DiagnosticComponent, ImplicitPrognosticComponent, Stepper + TendencyComponent, DiagnosticComponent, ImplicitTendencyComponent, Stepper ) @@ -10,7 +10,7 @@ class ScalingWrapper(object): Example ------- - This is how the ScalingWrapper can be used to wrap a PrognosticComponent. + This is how the ScalingWrapper can be used to wrap a TendencyComponent. >>> scaled_component = ScalingWrapper( >>> RRTMRadiation(), >>> input_scale_factors = { @@ -30,7 +30,7 @@ def __init__(self, Args ---- - component : PrognosticComponent, Stepper, DiagnosticComponent, ImplicitPrognosticComponent + component : TendencyComponent, Stepper, DiagnosticComponent, ImplicitTendencyComponent The component to be wrapped. input_scale_factors : dict a dictionary whose keys are the inputs that will be scaled @@ -53,17 +53,17 @@ def __init__(self, Raises ------ TypeError - The component is not of type Stepper or PrognosticComponent. + The component is not of type Stepper or TendencyComponent. ValueError The keys in the scale factors do not correspond to valid input/output/tendency for this component. """ if not any( isinstance(component, t) for t in [ - DiagnosticComponent, PrognosticComponent, ImplicitPrognosticComponent, Stepper]): + DiagnosticComponent, TendencyComponent, ImplicitTendencyComponent, Stepper]): raise TypeError( - 'component must be a component type (DiagnosticComponent, PrognosticComponent, ' - 'ImplicitPrognosticComponent, or Stepper)' + 'component must be a component type (DiagnosticComponent, TendencyComponent, ' + 'ImplicitTendencyComponent, or Stepper)' ) self._component = component @@ -158,7 +158,7 @@ def __call__(self, state, timestep=None): scale_factor = self._diagnostic_scale_factors[name] diagnostics[name] *= float(scale_factor) return diagnostics, new_state - elif isinstance(self._component, PrognosticComponent): + elif isinstance(self._component, TendencyComponent): tendencies, diagnostics = self._component(scaled_state) for tend_field in self._tendency_scale_factors.keys(): scale_factor = self._tendency_scale_factors[tend_field] @@ -167,9 +167,9 @@ def __call__(self, state, timestep=None): scale_factor = self._diagnostic_scale_factors[name] diagnostics[name] *= float(scale_factor) return tendencies, diagnostics - elif isinstance(self._component, ImplicitPrognosticComponent): + elif isinstance(self._component, ImplicitTendencyComponent): if timestep is None: - raise TypeError('Must give timestep to call ImplicitPrognosticComponent.') + raise TypeError('Must give timestep to call ImplicitTendencyComponent.') tendencies, diagnostics = self._component(scaled_state, timestep) for tend_field in self._tendency_scale_factors.keys(): scale_factor = self._tendency_scale_factors[tend_field] @@ -197,7 +197,7 @@ class UpdateFrequencyWrapper(object): Example ------- - This how the wrapper should be used on a fictional PrognosticComponent class + This how the wrapper should be used on a fictional TendencyComponent class called MyPrognostic. >>> from datetime import timedelta >>> prognostic = UpdateFrequencyWrapper(MyPrognostic(), timedelta(hours=1)) @@ -209,7 +209,7 @@ def __init__(self, component, update_timedelta): Args ---- - component : PrognosticComponent, Stepper, DiagnosticComponent, ImplicitPrognosticComponent + component : TendencyComponent, Stepper, DiagnosticComponent, ImplicitTendencyComponent The component to be wrapped. update_timedelta : timedelta The amount that state['time'] must differ from when output diff --git a/tests/test_base_components.py b/tests/test_base_components.py index 01dae41..fa9c547 100644 --- a/tests/test_base_components.py +++ b/tests/test_base_components.py @@ -3,7 +3,7 @@ import numpy as np import unittest from sympl import ( - PrognosticComponent, DiagnosticComponent, Monitor, Stepper, ImplicitPrognosticComponent, + TendencyComponent, DiagnosticComponent, Monitor, Stepper, ImplicitTendencyComponent, datetime, timedelta, DataArray, InvalidPropertyDictError, ComponentMissingOutputError, ComponentExtraOutputError, InvalidStateError @@ -14,7 +14,7 @@ def same_list(list1, list2): [item in list2 for item in list1] + [item in list1 for item in list2])) -class MockPrognosticComponent(PrognosticComponent): +class MockTendencyComponent(TendencyComponent): input_properties = None diagnostic_properties = None @@ -30,7 +30,7 @@ def __init__( self.tendency_output = tendency_output self.times_called = 0 self.state_given = None - super(MockPrognosticComponent, self).__init__(**kwargs) + super(MockTendencyComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -38,7 +38,7 @@ def array_call(self, state): return self.tendency_output, self.diagnostic_output -class MockImplicitPrognosticComponent(ImplicitPrognosticComponent): +class MockImplicitTendencyComponent(ImplicitTendencyComponent): input_properties = None diagnostic_properties = None @@ -55,7 +55,7 @@ def __init__( self.times_called = 0 self.state_given = None self.timestep_given = None - super(MockImplicitPrognosticComponent, self).__init__(**kwargs) + super(MockImplicitTendencyComponent, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -117,7 +117,7 @@ class MockMonitor(Monitor): def store(self, state): return -class BadMockPrognosticComponent(PrognosticComponent): +class BadMockTendencyComponent(TendencyComponent): input_properties = {} tendency_properties = {} @@ -130,7 +130,7 @@ def array_call(self, state): return {}, {} -class BadMockImplicitPrognosticComponent(ImplicitPrognosticComponent): +class BadMockImplicitTendencyComponent(ImplicitTendencyComponent): input_properties = {} tendency_properties = {} @@ -733,7 +733,7 @@ def test_raises_when_extraneous_diagnostic_given(self): class PrognosticTests(unittest.TestCase, InputTestBase): - component_class = MockPrognosticComponent + component_class = MockTendencyComponent def call_component(self, component, state): return component(state) @@ -742,7 +742,7 @@ def get_component( self, input_properties=None, tendency_properties=None, diagnostic_properties=None, tendency_output=None, diagnostic_output=None): - return MockPrognosticComponent( + return MockTendencyComponent( input_properties=input_properties or {}, tendency_properties=tendency_properties or {}, diagnostic_properties=diagnostic_properties or {}, @@ -758,7 +758,7 @@ def test_raises_on_tendency_properties_of_wrong_type(self): self.get_component(tendency_properties=({},)) def test_cannot_use_bad_component(self): - component = BadMockPrognosticComponent() + component = BadMockTendencyComponent() with self.assertRaises(RuntimeError): self.call_component(component, {'time': timedelta(0)}) @@ -775,7 +775,7 @@ def array_call(self): pass instance = MyPrognostic() - assert isinstance(instance, PrognosticComponent) + assert isinstance(instance, TendencyComponent) def test_tendency_raises_when_units_incompatible_with_input(self): input_properties = { @@ -791,7 +791,7 @@ def test_tendency_raises_when_units_incompatible_with_input(self): ) def test_two_components_are_not_instances_of_each_other(self): - class MyPrognosticComponent1(PrognosticComponent): + class MyTendencyComponent1(TendencyComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -800,7 +800,7 @@ class MyPrognosticComponent1(PrognosticComponent): def array_call(self, state): pass - class MyPrognosticComponent2(PrognosticComponent): + class MyTendencyComponent2(TendencyComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -809,10 +809,10 @@ class MyPrognosticComponent2(PrognosticComponent): def array_call(self, state): pass - prog1 = MyPrognosticComponent1() - prog2 = MyPrognosticComponent2() - assert not isinstance(prog1, MyPrognosticComponent2) - assert not isinstance(prog2, MyPrognosticComponent1) + prog1 = MyTendencyComponent1() + prog2 = MyTendencyComponent2() + assert not isinstance(prog1, MyTendencyComponent2) + assert not isinstance(prog2, MyTendencyComponent1) def test_ducktype_not_instance_of_subclass(self): class MyPrognostic1(object): @@ -826,7 +826,7 @@ def __init__(self): def array_call(self, state): pass - class MyPrognosticComponent2(PrognosticComponent): + class MyTendencyComponent2(TendencyComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -836,7 +836,7 @@ def array_call(self, state): pass prog1 = MyPrognostic1() - assert not isinstance(prog1, MyPrognosticComponent2) + assert not isinstance(prog1, MyTendencyComponent2) def test_empty_prognostic(self): prognostic = self.component_class({}, {}, {}, {}, {}) @@ -1289,7 +1289,7 @@ def test_tendencies_in_diagnostics_one_tendency_with_component_name(self): class ImplicitPrognosticTests(PrognosticTests): - component_class = MockImplicitPrognosticComponent + component_class = MockImplicitTendencyComponent def call_component(self, component, state): return component(state, timedelta(seconds=1)) @@ -1298,7 +1298,7 @@ def get_component( self, input_properties=None, tendency_properties=None, diagnostic_properties=None, tendency_output=None, diagnostic_output=None): - return MockImplicitPrognosticComponent( + return MockImplicitTendencyComponent( input_properties=input_properties or {}, tendency_properties=tendency_properties or {}, diagnostic_properties=diagnostic_properties or {}, @@ -1307,7 +1307,7 @@ def get_component( ) def test_cannot_use_bad_component(self): - component = BadMockImplicitPrognosticComponent() + component = BadMockImplicitTendencyComponent() with self.assertRaises(RuntimeError): self.call_component(component, {'time': timedelta(0)}) @@ -1324,10 +1324,10 @@ def array_call(self, state, timestep): pass instance = MyImplicitPrognostic() - assert isinstance(instance, ImplicitPrognosticComponent) + assert isinstance(instance, ImplicitTendencyComponent) def test_two_components_are_not_instances_of_each_other(self): - class MyImplicitPrognosticComponent1(ImplicitPrognosticComponent): + class MyImplicitTendencyComponent1(ImplicitTendencyComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -1337,7 +1337,7 @@ class MyImplicitPrognosticComponent1(ImplicitPrognosticComponent): def array_call(self, state, timestep): pass - class MyImplicitPrognosticComponent2(ImplicitPrognosticComponent): + class MyImplicitTendencyComponent2(ImplicitTendencyComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -1347,10 +1347,10 @@ class MyImplicitPrognosticComponent2(ImplicitPrognosticComponent): def array_call(self, state): pass - prog1 = MyImplicitPrognosticComponent1() - prog2 = MyImplicitPrognosticComponent2() - assert not isinstance(prog1, MyImplicitPrognosticComponent2) - assert not isinstance(prog2, MyImplicitPrognosticComponent1) + prog1 = MyImplicitTendencyComponent1() + prog2 = MyImplicitTendencyComponent2() + assert not isinstance(prog1, MyImplicitTendencyComponent2) + assert not isinstance(prog2, MyImplicitTendencyComponent1) def test_ducktype_not_instance_of_subclass(self): class MyImplicitPrognostic1(object): @@ -1364,7 +1364,7 @@ def __init__(self): def array_call(self, state, timestep): pass - class MyImplicitPrognosticComponent2(ImplicitPrognosticComponent): + class MyImplicitTendencyComponent2(ImplicitTendencyComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -1375,10 +1375,10 @@ def array_call(self, state): pass prog1 = MyImplicitPrognostic1() - assert not isinstance(prog1, MyImplicitPrognosticComponent2) + assert not isinstance(prog1, MyImplicitTendencyComponent2) def test_subclass_is_not_prognostic(self): - class MyImplicitPrognosticComponent(ImplicitPrognosticComponent): + class MyImplicitTendencyComponent(ImplicitTendencyComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} @@ -1387,8 +1387,8 @@ class MyImplicitPrognosticComponent(ImplicitPrognosticComponent): def array_call(self, state, timestep): pass - instance = MyImplicitPrognosticComponent() - assert not isinstance(instance, PrognosticComponent) + instance = MyImplicitTendencyComponent() + assert not isinstance(instance, TendencyComponent) def test_ducktype_is_not_prognostic(self): class MyImplicitPrognostic(object): @@ -1403,10 +1403,10 @@ def array_call(self, state, timestep): pass instance = MyImplicitPrognostic() - assert not isinstance(instance, PrognosticComponent) + assert not isinstance(instance, TendencyComponent) def test_timedelta_is_passed(self): - prognostic = MockImplicitPrognosticComponent({}, {}, {}, {}, {}) + prognostic = MockImplicitTendencyComponent({}, {}, {}, {}, {}) tendencies, diagnostics = prognostic( {'time': timedelta(seconds=0)}, timedelta(seconds=5)) assert tendencies == {} diff --git a/tests/test_components.py b/tests/test_components.py index 5bd5d33..7267cf4 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -1,13 +1,13 @@ import pytest from sympl import ( - ConstantPrognosticComponent, ConstantDiagnosticComponent, RelaxationPrognosticComponent, DataArray, + ConstantTendencyComponent, ConstantDiagnosticComponent, RelaxationTendencyComponent, DataArray, timedelta, ) import numpy as np def test_constant_prognostic_empty_dicts(): - prog = ConstantPrognosticComponent({}, {}) + prog = ConstantTendencyComponent({}, {}) tendencies, diagnostics = prog({'time': timedelta(0)}) assert isinstance(tendencies, dict) assert isinstance(diagnostics, dict) @@ -18,7 +18,7 @@ def test_constant_prognostic_empty_dicts(): def test_constant_prognostic_cannot_modify_through_input_dict(): in_tendencies = {} in_diagnostics = {} - prog = ConstantPrognosticComponent(in_tendencies, in_diagnostics) + prog = ConstantTendencyComponent(in_tendencies, in_diagnostics) in_tendencies['a'] = 'b' in_diagnostics['c'] = 'd' tendencies, diagnostics = prog({'time': timedelta(0)}) @@ -27,7 +27,7 @@ def test_constant_prognostic_cannot_modify_through_input_dict(): def test_constant_prognostic_cannot_modify_through_output_dict(): - prog = ConstantPrognosticComponent({}, {}) + prog = ConstantTendencyComponent({}, {}) tendencies, diagnostics = prog({'time': timedelta(0)}) tendencies['a'] = 'b' diagnostics['c'] = 'd' @@ -49,7 +49,7 @@ def test_constant_prognostic_tendency_properties(): attrs={'units': 'degK/s'}, ) } - prog = ConstantPrognosticComponent(tendencies) + prog = ConstantTendencyComponent(tendencies) assert prog.tendency_properties == { 'tend1': { 'dims': ('dim1',), @@ -78,7 +78,7 @@ def test_constant_prognostic_diagnostic_properties(): attrs={'units': 'degK'}, ) } - prog = ConstantPrognosticComponent(tendencies, diagnostics) + prog = ConstantTendencyComponent(tendencies, diagnostics) assert prog.diagnostic_properties == { 'diag1': { 'dims': ('dim1',), @@ -145,7 +145,7 @@ def test_constant_diagnostic_diagnostic_properties(): def test_relaxation_prognostic_at_equilibrium(): - prognostic = RelaxationPrognosticComponent('quantity', 'degK') + prognostic = RelaxationTendencyComponent('quantity', 'degK') state = { 'time': timedelta(0), 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'degK'}), @@ -159,7 +159,7 @@ def test_relaxation_prognostic_at_equilibrium(): def test_relaxation_prognostic_with_change(): - prognostic = RelaxationPrognosticComponent('quantity', 'degK') + prognostic = RelaxationTendencyComponent('quantity', 'degK') state = { 'time': timedelta(0), 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'degK'}), @@ -173,7 +173,7 @@ def test_relaxation_prognostic_with_change(): def test_relaxation_prognostic_with_change_different_timescale_units(): - prognostic = RelaxationPrognosticComponent('quantity', 'degK') + prognostic = RelaxationTendencyComponent('quantity', 'degK') state = { 'time': timedelta(0), 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'degK'}), @@ -187,7 +187,7 @@ def test_relaxation_prognostic_with_change_different_timescale_units(): def test_relaxation_prognostic_with_change_different_equilibrium_units(): - prognostic = RelaxationPrognosticComponent('quantity', 'm') + prognostic = RelaxationTendencyComponent('quantity', 'm') state = { 'time': timedelta(0), 'quantity': DataArray(np.array([0., 1., 2.]), attrs={'units': 'm'}), diff --git a/tests/test_composite.py b/tests/test_composite.py index e70a392..04658f7 100644 --- a/tests/test_composite.py +++ b/tests/test_composite.py @@ -2,7 +2,7 @@ import unittest import mock from sympl import ( - PrognosticComponent, DiagnosticComponent, Monitor, PrognosticComponentComposite, DiagnosticComponentComposite, + TendencyComponent, DiagnosticComponent, Monitor, TendencyComponentComposite, DiagnosticComponentComposite, MonitorComposite, SharedKeyError, DataArray, InvalidPropertyDictError ) from sympl._core.units import units_are_compatible @@ -13,7 +13,7 @@ def same_list(list1, list2): [item in list2 for item in list1] + [item in list1 for item in list2])) -class MockPrognosticComponent(PrognosticComponent): +class MockTendencyComponent(TendencyComponent): input_properties = None diagnostic_properties = None @@ -29,7 +29,7 @@ def __init__( self._tendency_output = tendency_output self.times_called = 0 self.state_given = None - super(MockPrognosticComponent, self).__init__(**kwargs) + super(MockTendencyComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -58,27 +58,27 @@ def array_call(self, state): return self._diagnostic_output -class MockEmptyPrognosticComponent(PrognosticComponent): +class MockEmptyTendencyComponent(TendencyComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} def __init__(self, **kwargs): - super(MockEmptyPrognosticComponent, self).__init__(**kwargs) + super(MockEmptyTendencyComponent, self).__init__(**kwargs) def array_call(self, state): return {}, {} -class MockEmptyPrognosticComponent2(PrognosticComponent): +class MockEmptyTendencyComponent2(TendencyComponent): input_properties = {} diagnostic_properties = {} tendency_properties = {} def __init__(self, **kwargs): - super(MockEmptyPrognosticComponent2, self).__init__(**kwargs) + super(MockEmptyTendencyComponent2, self).__init__(**kwargs) def array_call(self, state): return {}, {} @@ -103,7 +103,7 @@ def store(self, state): def test_empty_prognostic_composite(): - prognostic_composite = PrognosticComponentComposite() + prognostic_composite = TendencyComponentComposite() state = {'air_temperature': 273.15} tendencies, diagnostics = prognostic_composite(state) assert len(tendencies) == 0 @@ -112,10 +112,10 @@ def test_empty_prognostic_composite(): assert isinstance(diagnostics, dict) -@mock.patch.object(MockEmptyPrognosticComponent, '__call__') +@mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_prognostic_composite_calls_one_prognostic(mock_call): mock_call.return_value = ({'air_temperature': 0.5}, {'foo': 50.}) - prognostic_composite = PrognosticComponentComposite(MockEmptyPrognosticComponent()) + prognostic_composite = TendencyComponentComposite(MockEmptyTendencyComponent()) state = {'air_temperature': 273.15} tendencies, diagnostics = prognostic_composite(state) assert mock_call.called @@ -123,11 +123,11 @@ def test_prognostic_composite_calls_one_prognostic(mock_call): assert diagnostics == {'foo': 50.} -@mock.patch.object(MockEmptyPrognosticComponent, '__call__') +@mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_prognostic_composite_calls_two_prognostics(mock_call): mock_call.return_value = ({'air_temperature': 0.5}, {}) - prognostic_composite = PrognosticComponentComposite( - MockEmptyPrognosticComponent(), MockEmptyPrognosticComponent()) + prognostic_composite = TendencyComponentComposite( + MockEmptyTendencyComponent(), MockEmptyTendencyComponent()) state = {'air_temperature': 273.15} tendencies, diagnostics = prognostic_composite(state) assert mock_call.called @@ -182,7 +182,7 @@ def test_monitor_collection_calls_two_monitors(mock_store): def test_prognostic_composite_cannot_use_diagnostic(): try: - PrognosticComponentComposite(MockEmptyDiagnosticComponent()) + TendencyComponentComposite(MockEmptyDiagnosticComponent()) except TypeError: pass except Exception as err: @@ -193,7 +193,7 @@ def test_prognostic_composite_cannot_use_diagnostic(): def test_diagnostic_composite_cannot_use_prognostic(): try: - DiagnosticComponentComposite(MockEmptyPrognosticComponent()) + DiagnosticComponentComposite(MockEmptyTendencyComponent()) except TypeError: pass except Exception as err: @@ -214,16 +214,16 @@ def test_diagnostic_composite_call(mock_call): assert new_state['foo'] == 5. -@mock.patch.object(MockEmptyPrognosticComponent, '__call__') -@mock.patch.object(MockEmptyPrognosticComponent2, '__call__') +@mock.patch.object(MockEmptyTendencyComponent, '__call__') +@mock.patch.object(MockEmptyTendencyComponent2, '__call__') def test_prognostic_component_handles_units_when_combining(mock_call, mock2_call): mock_call.return_value = ({ 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})}, {}) mock2_call.return_value = ({ 'eastward_wind': DataArray(50., attrs={'units': 'cm/s'})}, {}) - prognostic1 = MockEmptyPrognosticComponent() - prognostic2 = MockEmptyPrognosticComponent2() - composite = PrognosticComponentComposite(prognostic1, prognostic2) + prognostic1 = MockEmptyTendencyComponent() + prognostic2 = MockEmptyTendencyComponent2() + composite = TendencyComponentComposite(prognostic1, prognostic2) tendencies, diagnostics = composite({}) assert tendencies['eastward_wind'].to_units('m/s').values.item() == 1.5 @@ -554,7 +554,7 @@ def test_diagnostic_composite_two_components_incompatible_input_units(): def test_prognostic_composite_single_input(): - prognostic = MockPrognosticComponent( + prognostic = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1'], @@ -566,14 +566,14 @@ def test_prognostic_composite_single_input(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic) + composite = TendencyComponentComposite(prognostic) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_prognostic_composite_single_diagnostic(): - prognostic = MockPrognosticComponent( + prognostic = MockTendencyComponent( input_properties={}, diagnostic_properties={ 'diag1': { @@ -585,14 +585,14 @@ def test_prognostic_composite_single_diagnostic(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic) + composite = TendencyComponentComposite(prognostic) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_prognostic_composite_single_tendency(): - prognostic = MockPrognosticComponent( + prognostic = MockTendencyComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -604,14 +604,14 @@ def test_prognostic_composite_single_tendency(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic) + composite = TendencyComponentComposite(prognostic) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_prognostic_composite_implicit_dims(): - prognostic = MockPrognosticComponent( + prognostic = MockTendencyComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -627,7 +627,7 @@ def test_prognostic_composite_implicit_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic) + composite = TendencyComponentComposite(prognostic) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == { @@ -639,7 +639,7 @@ def test_prognostic_composite_implicit_dims(): def test_two_prognostic_composite_implicit_dims(): - prognostic = MockPrognosticComponent( + prognostic = MockTendencyComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -655,7 +655,7 @@ def test_two_prognostic_composite_implicit_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -671,7 +671,7 @@ def test_two_prognostic_composite_implicit_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic, prognostic2) + composite = TendencyComponentComposite(prognostic, prognostic2) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == { @@ -683,7 +683,7 @@ def test_two_prognostic_composite_implicit_dims(): def test_prognostic_composite_explicit_dims(): - prognostic = MockPrognosticComponent( + prognostic = MockTendencyComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -700,14 +700,14 @@ def test_prognostic_composite_explicit_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic) + composite = TendencyComponentComposite(prognostic) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_two_prognostic_composite_explicit_dims(): - prognostic = MockPrognosticComponent( + prognostic = MockTendencyComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -724,7 +724,7 @@ def test_two_prognostic_composite_explicit_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -741,14 +741,14 @@ def test_two_prognostic_composite_explicit_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic, prognostic2) + composite = TendencyComponentComposite(prognostic, prognostic2) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_two_prognostic_composite_explicit_and_implicit_dims(): - prognostic = MockPrognosticComponent( + prognostic = MockTendencyComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -765,7 +765,7 @@ def test_two_prognostic_composite_explicit_and_implicit_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={ 'tend1': { 'dims': ['dims1', 'dims2'], @@ -781,14 +781,14 @@ def test_two_prognostic_composite_explicit_and_implicit_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic, prognostic2) + composite = TendencyComponentComposite(prognostic, prognostic2) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_prognostic_composite_explicit_dims_not_in_input(): - prognostic = MockPrognosticComponent( + prognostic = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -805,14 +805,14 @@ def test_prognostic_composite_explicit_dims_not_in_input(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic) + composite = TendencyComponentComposite(prognostic) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_two_prognostic_composite_incompatible_dims(): - prognostic = MockPrognosticComponent( + prognostic = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -833,7 +833,7 @@ def test_two_prognostic_composite_incompatible_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -855,7 +855,7 @@ def test_two_prognostic_composite_incompatible_dims(): tendency_output={}, ) try: - PrognosticComponentComposite(prognostic, prognostic2) + TendencyComponentComposite(prognostic, prognostic2) except InvalidPropertyDictError: pass else: @@ -863,7 +863,7 @@ def test_two_prognostic_composite_incompatible_dims(): def test_two_prognostic_composite_compatible_dims(): - prognostic = MockPrognosticComponent( + prognostic = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -884,7 +884,7 @@ def test_two_prognostic_composite_compatible_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -905,14 +905,14 @@ def test_two_prognostic_composite_compatible_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic, prognostic2) + composite = TendencyComponentComposite(prognostic, prognostic2) assert composite.input_properties == prognostic.input_properties assert composite.diagnostic_properties == prognostic.diagnostic_properties assert composite.tendency_properties == prognostic.tendency_properties def test_prognostic_composite_two_components_input(): - prognostic1 = MockPrognosticComponent( + prognostic1 = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -928,7 +928,7 @@ def test_prognostic_composite_two_components_input(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -944,7 +944,7 @@ def test_prognostic_composite_two_components_input(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic1, prognostic2) + composite = TendencyComponentComposite(prognostic1, prognostic2) input_properties = { 'input1': { 'dims': ['dims1', 'dims2'], @@ -967,7 +967,7 @@ def test_prognostic_composite_two_components_input(): def test_prognostic_composite_two_components_swapped_input_dims(): - prognostic1 = MockPrognosticComponent( + prognostic1 = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -979,7 +979,7 @@ def test_prognostic_composite_two_components_swapped_input_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims2', 'dims1'], @@ -991,7 +991,7 @@ def test_prognostic_composite_two_components_swapped_input_dims(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic1, prognostic2) + composite = TendencyComponentComposite(prognostic1, prognostic2) diagnostic_properties = {} tendency_properties = {} assert (composite.input_properties == prognostic1.input_properties or @@ -1001,7 +1001,7 @@ def test_prognostic_composite_two_components_swapped_input_dims(): def test_prognostic_composite_two_components_incompatible_input_dims(): - prognostic1 = MockPrognosticComponent( + prognostic1 = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -1013,7 +1013,7 @@ def test_prognostic_composite_two_components_incompatible_input_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims3'], @@ -1026,7 +1026,7 @@ def test_prognostic_composite_two_components_incompatible_input_dims(): tendency_output={}, ) try: - PrognosticComponentComposite(prognostic1, prognostic2) + TendencyComponentComposite(prognostic1, prognostic2) except InvalidPropertyDictError: pass else: @@ -1034,7 +1034,7 @@ def test_prognostic_composite_two_components_incompatible_input_dims(): def test_prognostic_composite_two_components_incompatible_input_units(): - prognostic1 = MockPrognosticComponent( + prognostic1 = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -1046,7 +1046,7 @@ def test_prognostic_composite_two_components_incompatible_input_units(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -1059,7 +1059,7 @@ def test_prognostic_composite_two_components_incompatible_input_units(): tendency_output={}, ) try: - PrognosticComponentComposite(prognostic1, prognostic2) + TendencyComponentComposite(prognostic1, prognostic2) except InvalidPropertyDictError: pass else: @@ -1067,7 +1067,7 @@ def test_prognostic_composite_two_components_incompatible_input_units(): def test_prognostic_composite_two_components_compatible_input_units(): - prognostic1 = MockPrognosticComponent( + prognostic1 = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -1079,7 +1079,7 @@ def test_prognostic_composite_two_components_compatible_input_units(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={ 'input1': { 'dims': ['dims1', 'dims2'], @@ -1091,14 +1091,14 @@ def test_prognostic_composite_two_components_compatible_input_units(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic1, prognostic2) + composite = TendencyComponentComposite(prognostic1, prognostic2) assert 'input1' in composite.input_properties.keys() assert composite.input_properties['input1']['dims'] == ['dims1', 'dims2'] assert units_are_compatible(composite.input_properties['input1']['units'], 'm') def test_prognostic_composite_two_components_tendency(): - prognostic1 = MockPrognosticComponent( + prognostic1 = MockTendencyComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1110,7 +1110,7 @@ def test_prognostic_composite_two_components_tendency(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1126,7 +1126,7 @@ def test_prognostic_composite_two_components_tendency(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic1, prognostic2) + composite = TendencyComponentComposite(prognostic1, prognostic2) input_properties = {} diagnostic_properties = {} tendency_properties = { @@ -1145,7 +1145,7 @@ def test_prognostic_composite_two_components_tendency(): def test_prognostic_composite_two_components_tendency_incompatible_dims(): - prognostic1 = MockPrognosticComponent( + prognostic1 = MockTendencyComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1157,7 +1157,7 @@ def test_prognostic_composite_two_components_tendency_incompatible_dims(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1174,7 +1174,7 @@ def test_prognostic_composite_two_components_tendency_incompatible_dims(): tendency_output={}, ) try: - PrognosticComponentComposite(prognostic1, prognostic2) + TendencyComponentComposite(prognostic1, prognostic2) except InvalidPropertyDictError: pass else: @@ -1182,7 +1182,7 @@ def test_prognostic_composite_two_components_tendency_incompatible_dims(): def test_prognostic_composite_two_components_tendency_incompatible_units(): - prognostic1 = MockPrognosticComponent( + prognostic1 = MockTendencyComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1194,7 +1194,7 @@ def test_prognostic_composite_two_components_tendency_incompatible_units(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1211,7 +1211,7 @@ def test_prognostic_composite_two_components_tendency_incompatible_units(): tendency_output={}, ) try: - PrognosticComponentComposite(prognostic1, prognostic2) + TendencyComponentComposite(prognostic1, prognostic2) except InvalidPropertyDictError: pass else: @@ -1219,7 +1219,7 @@ def test_prognostic_composite_two_components_tendency_incompatible_units(): def test_prognostic_composite_two_components_tendency_compatible_units(): - prognostic1 = MockPrognosticComponent( + prognostic1 = MockTendencyComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1231,7 +1231,7 @@ def test_prognostic_composite_two_components_tendency_compatible_units(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={}, diagnostic_properties={}, tendency_properties={ @@ -1243,14 +1243,14 @@ def test_prognostic_composite_two_components_tendency_compatible_units(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic1, prognostic2) + composite = TendencyComponentComposite(prognostic1, prognostic2) assert 'tend1' in composite.tendency_properties.keys() assert composite.tendency_properties['tend1']['dims'] == ['dim1'] assert units_are_compatible(composite.tendency_properties['tend1']['units'], 'm/s') def test_prognostic_composite_two_components_diagnostic(): - prognostic1 = MockPrognosticComponent( + prognostic1 = MockTendencyComponent( input_properties={}, diagnostic_properties={ 'diag1': { @@ -1262,7 +1262,7 @@ def test_prognostic_composite_two_components_diagnostic(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={}, diagnostic_properties={ 'diag2': { @@ -1274,7 +1274,7 @@ def test_prognostic_composite_two_components_diagnostic(): diagnostic_output={}, tendency_output={}, ) - composite = PrognosticComponentComposite(prognostic1, prognostic2) + composite = TendencyComponentComposite(prognostic1, prognostic2) input_properties = {} diagnostic_properties = { 'diag1': { @@ -1293,7 +1293,7 @@ def test_prognostic_composite_two_components_diagnostic(): def test_prognostic_composite_two_components_overlapping_diagnostic(): - prognostic1 = MockPrognosticComponent( + prognostic1 = MockTendencyComponent( input_properties={}, diagnostic_properties={ 'diag1': { @@ -1305,7 +1305,7 @@ def test_prognostic_composite_two_components_overlapping_diagnostic(): diagnostic_output={}, tendency_output={}, ) - prognostic2 = MockPrognosticComponent( + prognostic2 = MockTendencyComponent( input_properties={}, diagnostic_properties={ 'diag1': { @@ -1318,7 +1318,7 @@ def test_prognostic_composite_two_components_overlapping_diagnostic(): tendency_output={}, ) try: - PrognosticComponentComposite(prognostic1, prognostic2) + TendencyComponentComposite(prognostic1, prognostic2) except SharedKeyError: pass else: diff --git a/tests/test_time_differencing_wrapper.py b/tests/test_time_differencing_wrapper.py index a7daaa7..3cd4499 100644 --- a/tests/test_time_differencing_wrapper.py +++ b/tests/test_time_differencing_wrapper.py @@ -1,14 +1,14 @@ from datetime import timedelta, datetime import unittest from sympl import ( - PrognosticComponent, Stepper, DiagnosticComponent, TimeDifferencingWrapper, DataArray + TendencyComponent, Stepper, DiagnosticComponent, TimeDifferencingWrapper, DataArray ) import pytest from numpy.testing import assert_allclose from copy import deepcopy -class MockPrognosticComponent(PrognosticComponent): +class MockTendencyComponent(TendencyComponent): def __init__(self): self._num_updates = 0 @@ -68,7 +68,7 @@ def __call__(self, state, timestep): return deepcopy(state), state -class MockPrognosticComponentThatExpects(PrognosticComponent): +class MockTendencyComponentThatExpects(TendencyComponent): input_properties = {'expected_field': {}} tendency_properties = {'expected_field': {}} diff --git a/tests/test_timestepping.py b/tests/test_timestepping.py index 9fdece4..fb202dc 100644 --- a/tests/test_timestepping.py +++ b/tests/test_timestepping.py @@ -1,8 +1,8 @@ import pytest import mock from sympl import ( - PrognosticComponent, Leapfrog, AdamsBashforth, DataArray, SSPRungeKutta, timedelta, - InvalidPropertyDictError, ImplicitPrognosticComponent) + TendencyComponent, Leapfrog, AdamsBashforth, DataArray, SSPRungeKutta, timedelta, + InvalidPropertyDictError, ImplicitTendencyComponent) from sympl._core.units import units_are_compatible import numpy as np import warnings @@ -13,7 +13,7 @@ def same_list(list1, list2): [item in list2 for item in list1] + [item in list1 for item in list2])) -class MockEmptyPrognosticComponent(PrognosticComponent): +class MockEmptyTendencyComponent(TendencyComponent): input_properties = {} tendency_properties = {} @@ -23,7 +23,7 @@ def array_call(self, state): return {}, {} -class MockPrognosticComponent(PrognosticComponent): +class MockTendencyComponent(TendencyComponent): input_properties = None diagnostic_properties = None @@ -39,7 +39,7 @@ def __init__( self._tendency_output = tendency_output self.times_called = 0 self.state_given = None - super(MockPrognosticComponent, self).__init__(**kwargs) + super(MockTendencyComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -47,7 +47,7 @@ def array_call(self, state): return self._tendency_output, self._diagnostic_output -class MockImplicitPrognosticComponent(ImplicitPrognosticComponent): +class MockImplicitTendencyComponent(ImplicitTendencyComponent): input_properties = None diagnostic_properties = None tendency_properties = None @@ -62,7 +62,7 @@ def __init__( self._tendency_output = tendency_output self.times_called = 0 self.state_given = None - super(MockImplicitPrognosticComponent, self).__init__(**kwargs) + super(MockImplicitTendencyComponent, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -72,7 +72,7 @@ def array_call(self, state, timestep): class PrognosticBase(object): - prognostic_class = MockPrognosticComponent + prognostic_class = MockTendencyComponent def test_given_tendency_not_modified_with_two_components(self): input_properties = {} @@ -408,7 +408,7 @@ def test_stepper_gives_diagnostic_tendency_quantity(self): class ImplicitPrognosticBase(PrognosticBase): - prognostic_class = MockImplicitPrognosticComponent + prognostic_class = MockImplicitTendencyComponent class TimesteppingBase(object): @@ -417,47 +417,47 @@ class TimesteppingBase(object): def test_unused_quantities_carried_over(self): state = {'time': timedelta(0), 'air_temperature': 273.} - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) timestep = timedelta(seconds=1.) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} def test_timestepper_reveals_prognostics(self): - prog1 = MockEmptyPrognosticComponent() + prog1 = MockEmptyTendencyComponent() prog1.input_properties = {'input1': {'dims': ['dim1'], 'units': 'm'}} time_stepper = self.timestepper_class(prog1) assert same_list(time_stepper.prognostic_list, (prog1,)) - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_float_no_change_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} assert diagnostics == {} - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_float_no_change_one_step_diagnostic(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': 0.}, {'foo': 'bar'}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} assert diagnostics == {'foo': 'bar'} - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_float_no_change_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} @@ -470,34 +470,34 @@ def test_float_no_change_three_steps(self, mock_prognostic_call): assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_float_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 1.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 274.} - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_float_one_step_with_units(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'eastward_wind': DataArray(0.02, attrs={'units': 'km/s^2'})}, {}) state = {'time': timedelta(0), 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'eastward_wind': DataArray(1., attrs={'units': 'm/s'})} assert same_list(new_state.keys(), ['time', 'eastward_wind']) assert np.allclose(new_state['eastward_wind'].values, 21.) assert new_state['eastward_wind'].attrs['units'] == 'm/s' - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_float_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ({'air_temperature': 1.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 274.} @@ -510,26 +510,26 @@ def test_float_three_steps(self, mock_prognostic_call): assert state == {'time': timedelta(0), 'air_temperature': 275.} assert new_state == {'time': timedelta(0), 'air_temperature': 276.} - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_array_no_change_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.zeros((3, 3))}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_array_no_change_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*0.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -548,26 +548,26 @@ def test_array_no_change_three_steps(self, mock_prognostic_call): assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*273.).all() - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_array_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*1.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*274.).all() - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_array_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*1.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -586,7 +586,7 @@ def test_array_three_steps(self, mock_prognostic_call): assert same_list(new_state.keys(), ['time', 'air_temperature']) assert (new_state['air_temperature'] == np.ones((3, 3))*276.).all() - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_dataarray_no_change_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': @@ -595,7 +595,7 @@ def test_dataarray_no_change_one_step(self, mock_prognostic_call): state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'].values == np.ones((3, 3))*273.).all() @@ -606,7 +606,7 @@ def test_dataarray_no_change_one_step(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_dataarray_no_change_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': @@ -615,7 +615,7 @@ def test_dataarray_no_change_three_steps(self, mock_prognostic_call): state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -642,7 +642,7 @@ def test_dataarray_no_change_three_steps(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_dataarray_one_step(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': @@ -651,7 +651,7 @@ def test_dataarray_one_step(self, mock_prognostic_call): state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -662,7 +662,7 @@ def test_dataarray_one_step(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_dataarray_three_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': @@ -671,7 +671,7 @@ def test_dataarray_three_steps(self, mock_prognostic_call): state = {'time': timedelta(0), 'air_temperature': DataArray(np.ones((3, 3))*273., attrs={'units': 'K'})} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -698,13 +698,13 @@ def test_dataarray_three_steps(self, mock_prognostic_call): assert len(new_state['air_temperature'].attrs) == 1 assert new_state['air_temperature'].attrs['units'] == 'K' - @mock.patch.object(MockEmptyPrognosticComponent, '__call__') + @mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_array_four_steps(self, mock_prognostic_call): mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3)) * 1.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3)) * 273.} timestep = timedelta(seconds=1.) - time_stepper = self.timestepper_class(MockEmptyPrognosticComponent()) + time_stepper = self.timestepper_class(MockEmptyTendencyComponent()) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3)) * 273.).all() @@ -823,13 +823,13 @@ def timestepper_class(self, *args, **kwargs): return AdamsBashforth(*args, **kwargs) -@mock.patch.object(MockEmptyPrognosticComponent, '__call__') +@mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_leapfrog_float_two_steps_filtered(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} timestep = timedelta(seconds=1.) - time_stepper = Leapfrog(MockEmptyPrognosticComponent(), asselin_strength=0.5, alpha=1.) + time_stepper = Leapfrog(MockEmptyTendencyComponent(), asselin_strength=0.5, alpha=1.) diagnostics, new_state = time_stepper.__call__(state, timestep) assert state == {'time': timedelta(0), 'air_temperature': 273.} assert new_state == {'time': timedelta(0), 'air_temperature': 273.} @@ -841,12 +841,12 @@ def test_leapfrog_float_two_steps_filtered(mock_prognostic_call): assert new_state == {'time': timedelta(0), 'air_temperature': 277.} -@mock.patch.object(MockEmptyPrognosticComponent, '__call__') +@mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_leapfrog_requires_same_timestep(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} - time_stepper = Leapfrog([MockEmptyPrognosticComponent()], asselin_strength=0.5) + time_stepper = Leapfrog([MockEmptyTendencyComponent()], asselin_strength=0.5) diagnostics, state = time_stepper.__call__(state, timedelta(seconds=1.)) try: time_stepper.__call__(state, timedelta(seconds=2.)) @@ -858,12 +858,12 @@ def test_leapfrog_requires_same_timestep(mock_prognostic_call): raise AssertionError('Leapfrog must require timestep to be constant') -@mock.patch.object(MockEmptyPrognosticComponent, '__call__') +@mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_adams_bashforth_requires_same_timestep(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ({'air_temperature': 0.}, {}) state = {'time': timedelta(0), 'air_temperature': 273.} - time_stepper = AdamsBashforth(MockEmptyPrognosticComponent()) + time_stepper = AdamsBashforth(MockEmptyTendencyComponent()) state = time_stepper.__call__(state, timedelta(seconds=1.)) try: time_stepper.__call__(state, timedelta(seconds=2.)) @@ -876,14 +876,14 @@ def test_adams_bashforth_requires_same_timestep(mock_prognostic_call): 'AdamsBashforth must require timestep to be constant') -@mock.patch.object(MockEmptyPrognosticComponent, '__call__') +@mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_leapfrog_array_two_steps_filtered(mock_prognostic_call): """Test that the Asselin filter is being correctly applied""" mock_prognostic_call.return_value = ( {'air_temperature': np.ones((3, 3))*0.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = Leapfrog(MockEmptyPrognosticComponent(), asselin_strength=0.5, alpha=1.) + time_stepper = Leapfrog(MockEmptyTendencyComponent(), asselin_strength=0.5, alpha=1.) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() @@ -900,7 +900,7 @@ def test_leapfrog_array_two_steps_filtered(mock_prognostic_call): assert (new_state['air_temperature'] == np.ones((3, 3))*277.).all() -@mock.patch.object(MockEmptyPrognosticComponent, '__call__') +@mock.patch.object(MockEmptyTendencyComponent, '__call__') def test_leapfrog_array_two_steps_filtered_williams(mock_prognostic_call): """Test that the Asselin filter is being correctly applied with a Williams factor of alpha=0.5""" @@ -908,7 +908,7 @@ def test_leapfrog_array_two_steps_filtered_williams(mock_prognostic_call): {'air_temperature': np.ones((3, 3))*0.}, {}) state = {'time': timedelta(0), 'air_temperature': np.ones((3, 3))*273.} timestep = timedelta(seconds=1.) - time_stepper = Leapfrog(MockEmptyPrognosticComponent(), asselin_strength=0.5, alpha=0.5) + time_stepper = Leapfrog(MockEmptyTendencyComponent(), asselin_strength=0.5, alpha=0.5) diagnostics, new_state = time_stepper.__call__(state, timestep) assert same_list(state.keys(), ['time', 'air_temperature']) assert (state['air_temperature'] == np.ones((3, 3))*273.).all() diff --git a/tests/test_tracers.py b/tests/test_tracers.py index f7c1448..9adbd68 100644 --- a/tests/test_tracers.py +++ b/tests/test_tracers.py @@ -1,6 +1,6 @@ from sympl._core.tracers import TracerPacker, clear_tracers, clear_packers from sympl import ( - PrognosticComponent, Stepper, DiagnosticComponent, ImplicitPrognosticComponent, register_tracer, + TendencyComponent, Stepper, DiagnosticComponent, ImplicitTendencyComponent, register_tracer, get_tracer_unit_dict, units_are_compatible, DataArray ) import unittest @@ -9,7 +9,7 @@ from datetime import timedelta -class MockPrognosticComponent(PrognosticComponent): +class MockTendencyComponent(TendencyComponent): input_properties = None diagnostic_properties = None @@ -23,7 +23,7 @@ def __init__(self, **kwargs): self.tendency_output = {} self.times_called = 0 self.state_given = None - super(MockPrognosticComponent, self).__init__(**kwargs) + super(MockTendencyComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -31,7 +31,7 @@ def array_call(self, state): return self.tendency_output, self.diagnostic_output -class MockTracerPrognosticComponent(PrognosticComponent): +class MockTracerTendencyComponent(TendencyComponent): input_properties = None diagnostic_properties = None @@ -50,7 +50,7 @@ def __init__(self, **kwargs): self.diagnostic_output = {} self.times_called = 0 self.state_given = None - super(MockTracerPrognosticComponent, self).__init__(**kwargs) + super(MockTracerTendencyComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -61,7 +61,7 @@ def array_call(self, state): return return_state, self.diagnostic_output -class MockImplicitPrognosticComponent(ImplicitPrognosticComponent): +class MockImplicitTendencyComponent(ImplicitTendencyComponent): input_properties = None diagnostic_properties = None @@ -76,7 +76,7 @@ def __init__( self, **kwargs): self.times_called = 0 self.state_given = None self.timestep_given = None - super(MockImplicitPrognosticComponent, self).__init__(**kwargs) + super(MockImplicitTendencyComponent, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -85,7 +85,7 @@ def array_call(self, state, timestep): return self.tendency_output, self.diagnostic_output -class MockTracerImplicitPrognosticComponent(ImplicitPrognosticComponent): +class MockTracerImplicitTendencyComponent(ImplicitTendencyComponent): input_properties = None diagnostic_properties = None @@ -105,7 +105,7 @@ def __init__(self, **kwargs): self.times_called = 0 self.state_given = None self.timestep_given = None - super(MockTracerImplicitPrognosticComponent, self).__init__(**kwargs) + super(MockTracerImplicitTendencyComponent, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -375,7 +375,7 @@ def test_unpacks_three_tracers_in_order_registered(self): class PrognosticTracerPackerTests(TracerPackerBase, unittest.TestCase): def setUp(self): - self.component = MockPrognosticComponent() + self.component = MockTendencyComponent() super(PrognosticTracerPackerTests, self).setUp() def tearDown(self): @@ -408,7 +408,7 @@ def test_packs_updates_tendency_properties_after_init(self): class ImplicitPrognosticTracerPackerTests(PrognosticTracerPackerTests): def setUp(self): - self.component = MockImplicitPrognosticComponent() + self.component = MockImplicitTendencyComponent() super(ImplicitPrognosticTracerPackerTests, self).setUp() def tearDown(self): @@ -702,7 +702,7 @@ class PrognosticTracerComponentTests(TracerComponentBase, unittest.TestCase): def setUp(self): super(PrognosticTracerComponentTests, self).setUp() - self.component = MockTracerPrognosticComponent() + self.component = MockTracerTendencyComponent() def tearDown(self): super(PrognosticTracerComponentTests, self).tearDown() @@ -716,7 +716,7 @@ class ImplicitPrognosticTracerComponentTests(TracerComponentBase, unittest.TestC def setUp(self): super(ImplicitPrognosticTracerComponentTests, self).setUp() - self.component = MockTracerImplicitPrognosticComponent() + self.component = MockTracerImplicitTendencyComponent() def tearDown(self): super(ImplicitPrognosticTracerComponentTests, self).tearDown() diff --git a/tests/test_util.py b/tests/test_util.py index 1d48922..5951161 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -2,7 +2,7 @@ import numpy as np import pytest from sympl import ( - PrognosticComponent, ensure_no_shared_keys, SharedKeyError, DataArray, + TendencyComponent, ensure_no_shared_keys, SharedKeyError, DataArray, Stepper, DiagnosticComponent, InvalidPropertyDictError) from sympl._core.util import update_dict_by_adding_another, combine_dims, get_component_aliases @@ -64,7 +64,7 @@ def test_update_dict_by_adding_another_adds_shared_arrays_reversed(): assert len(dict2.keys()) == 2 -class DummyPrognosticComponent(PrognosticComponent): +class DummyTendencyComponent(TendencyComponent): input_properties = {'temperature': {'alias': 'T'}} diagnostic_properties = {'pressure': {'alias': 'P'}} tendency_properties = {'temperature': {}} diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index 6d57122..96fc67c 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -1,14 +1,14 @@ from datetime import timedelta, datetime import unittest from sympl import ( - PrognosticComponent, Stepper, DiagnosticComponent, UpdateFrequencyWrapper, ScalingWrapper, - TimeDifferencingWrapper, DataArray, ImplicitPrognosticComponent + TendencyComponent, Stepper, DiagnosticComponent, UpdateFrequencyWrapper, ScalingWrapper, + TimeDifferencingWrapper, DataArray, ImplicitTendencyComponent ) import pytest import numpy as np -class MockPrognosticComponent(PrognosticComponent): +class MockTendencyComponent(TendencyComponent): input_properties = None diagnostic_properties = None @@ -24,7 +24,7 @@ def __init__( self.tendency_output = tendency_output self.times_called = 0 self.state_given = None - super(MockPrognosticComponent, self).__init__(**kwargs) + super(MockTendencyComponent, self).__init__(**kwargs) def array_call(self, state): self.times_called += 1 @@ -32,7 +32,7 @@ def array_call(self, state): return self.tendency_output, self.diagnostic_output -class MockImplicitPrognosticComponent(ImplicitPrognosticComponent): +class MockImplicitTendencyComponent(ImplicitTendencyComponent): input_properties = None diagnostic_properties = None @@ -49,7 +49,7 @@ def __init__( self.times_called = 0 self.state_given = None self.timestep_given = None - super(MockImplicitPrognosticComponent, self).__init__(**kwargs) + super(MockImplicitTendencyComponent, self).__init__(**kwargs) def array_call(self, state, timestep): self.times_called += 1 @@ -106,7 +106,7 @@ def array_call(self, state, timestep): return self.diagnostic_output, self.state_output -class MockEmptyPrognostic(MockPrognosticComponent): +class MockEmptyPrognostic(MockTendencyComponent): def __init__(self, **kwargs): super(MockEmptyPrognostic, self).__init__( @@ -119,7 +119,7 @@ def __init__(self, **kwargs): ) -class MockEmptyImplicitPrognostic(MockImplicitPrognosticComponent): +class MockEmptyImplicitPrognostic(MockImplicitTendencyComponent): def __init__(self, **kwargs): super(MockEmptyImplicitPrognostic, self).__init__( input_properties={}, @@ -210,7 +210,7 @@ def test_set_update_frequency_does_not_update_when_less(self): class PrognosticUpdateFrequencyTests(unittest.TestCase, UpdateFrequencyBase): - component_type = PrognosticComponent + component_type = TendencyComponent def get_component(self): return MockEmptyPrognostic() @@ -221,7 +221,7 @@ def call_component(self, component, state): class ImplicitPrognosticUpdateFrequencyTests(unittest.TestCase, UpdateFrequencyBase): - component_type = ImplicitPrognosticComponent + component_type = ImplicitTendencyComponent def get_component(self): return MockEmptyImplicitPrognostic() @@ -679,7 +679,7 @@ def call_component(self, component, state): class PrognosticScalingTests( unittest.TestCase, ScalingInputMixin, ScalingDiagnosticMixin, ScalingTendencyMixin): - component_type = PrognosticComponent + component_type = TendencyComponent def setUp(self): self.input_properties = {} @@ -689,7 +689,7 @@ def setUp(self): self.tendency_output = {} def get_component(self): - return MockPrognosticComponent( + return MockTendencyComponent( self.input_properties, self.diagnostic_properties, self.tendency_properties, @@ -711,7 +711,7 @@ class ImplicitPrognosticScalingTests( unittest.TestCase, ScalingInputMixin, ScalingDiagnosticMixin, ScalingTendencyMixin): - component_type = ImplicitPrognosticComponent + component_type = ImplicitTendencyComponent def setUp(self): self.input_properties = {} @@ -721,7 +721,7 @@ def setUp(self): self.tendency_output = {} def get_component(self): - return MockImplicitPrognosticComponent( + return MockImplicitTendencyComponent( self.input_properties, self.diagnostic_properties, self.tendency_properties, From 1b43f31a01d79d31b5851c0b2cee685db6d4659f Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Fri, 27 Jul 2018 17:11:11 -0700 Subject: [PATCH 84/98] renamed tracer/packer reset methods for consistency --- sympl/__init__.py | 3 ++- sympl/_core/tracers.py | 4 ++-- tests/test_tracers.py | 22 +++++++++++----------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/sympl/__init__.py b/sympl/__init__.py index 203a4d3..ecbcd22 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -3,7 +3,7 @@ TendencyComponent, DiagnosticComponent, Stepper, Monitor, ImplicitTendencyComponent ) from ._core.composite import TendencyComponentComposite, DiagnosticComponentComposite, \ - MonitorComposite + MonitorComposite, ImplicitTendencyComponentComposite from ._core.prognosticstepper import PrognosticStepper from ._components.timesteppers import AdamsBashforth, Leapfrog, SSPRungeKutta from ._core.exceptions import ( @@ -36,6 +36,7 @@ __version__ = '0.3.2' __all__ = ( TendencyComponent, DiagnosticComponent, Stepper, Monitor, TendencyComponentComposite, + ImplicitTendencyComponentComposite, DiagnosticComponentComposite, MonitorComposite, ImplicitTendencyComponent, PrognosticStepper, Leapfrog, AdamsBashforth, SSPRungeKutta, InvalidStateError, SharedKeyError, DependencyError, diff --git a/sympl/_core/tracers.py b/sympl/_core/tracers.py index 8c1805f..6b59930 100644 --- a/sympl/_core/tracers.py +++ b/sympl/_core/tracers.py @@ -7,7 +7,7 @@ _packers = set() -def clear_tracers(): +def reset_tracers(): global _tracer_names while len(_tracer_unit_dict) > 0: _tracer_unit_dict.popitem() @@ -15,7 +15,7 @@ def clear_tracers(): _tracer_names.pop() -def clear_packers(): +def reset_packers(): while len(_packers) > 0: _packers.pop() diff --git a/tests/test_tracers.py b/tests/test_tracers.py index 9adbd68..ee6fe3a 100644 --- a/tests/test_tracers.py +++ b/tests/test_tracers.py @@ -1,4 +1,4 @@ -from sympl._core.tracers import TracerPacker, clear_tracers, clear_packers +from sympl._core.tracers import TracerPacker, reset_tracers, reset_packers from sympl import ( TendencyComponent, Stepper, DiagnosticComponent, ImplicitTendencyComponent, register_tracer, get_tracer_unit_dict, units_are_compatible, DataArray @@ -196,10 +196,10 @@ def array_call(self, state, timestep): class RegisterTracerTests(unittest.TestCase): def setUp(self): - clear_tracers() + reset_tracers() def tearDown(self): - clear_tracers() + reset_tracers() def test_initially_empty(self): assert len(get_tracer_unit_dict()) == 0 @@ -238,12 +238,12 @@ def test_reregister_tracer_different_units(self): class TracerPackerBase(object): def setUp(self): - clear_tracers() - clear_packers() + reset_tracers() + reset_packers() def tearDown(self): - clear_tracers() - clear_packers() + reset_tracers() + reset_packers() def test_packs_no_tracers(self): dims = ['tracer', '*'] @@ -460,12 +460,12 @@ def test_raises_on_diagnostic_init(self): class TracerComponentBase(object): def setUp(self): - clear_tracers() - clear_packers() + reset_tracers() + reset_packers() def tearDown(self): - clear_tracers() - clear_packers() + reset_tracers() + reset_packers() def call_component(self, input_state): pass From 2cdabac789541064b8133dd5356de3620b40f34a Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Jul 2018 21:17:41 -0700 Subject: [PATCH 85/98] Removed circular dependency to finish tracer implementation --- sympl/__init__.py | 9 +- sympl/_components/basic.py | 2 +- sympl/_components/netcdf.py | 2 +- sympl/_components/plot.py | 2 +- sympl/_components/timesteppers.py | 2 +- sympl/_core/base_components.py | 59 ++-- sympl/_core/constants.py | 2 +- sympl/_core/{array.py => dataarray.py} | 2 +- sympl/_core/get_np_arrays.py | 68 +++++ sympl/_core/init_np_arrays.py | 78 ++++++ sympl/_core/restore_dataarray.py | 135 +++++++++ sympl/_core/state.py | 371 ------------------------- sympl/_core/tracers.py | 123 +++++--- sympl/_core/util.py | 2 +- sympl/_core/wildcard.py | 93 +++++++ tests/test_tracers.py | 254 ++++++++++------- 16 files changed, 653 insertions(+), 551 deletions(-) rename sympl/_core/{array.py => dataarray.py} (100%) create mode 100644 sympl/_core/get_np_arrays.py create mode 100644 sympl/_core/init_np_arrays.py create mode 100644 sympl/_core/restore_dataarray.py create mode 100644 sympl/_core/wildcard.py diff --git a/sympl/__init__.py b/sympl/__init__.py index ecbcd22..0550c8f 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -10,7 +10,7 @@ InvalidStateError, SharedKeyError, DependencyError, InvalidPropertyDictError, ComponentExtraOutputError, ComponentMissingOutputError) -from ._core.array import DataArray +from ._core.dataarray import DataArray from ._core.constants import ( get_constant, set_constant, set_condensible_name, reset_constants, get_constants_string) @@ -22,10 +22,9 @@ get_component_aliases, combine_component_properties) from ._core.units import units_are_same, units_are_compatible, is_valid_unit -from ._core.state import ( - get_numpy_arrays_with_properties, - restore_data_arrays_with_properties, - initialize_numpy_arrays_with_properties) +from sympl._core.get_np_arrays import get_numpy_arrays_with_properties +from sympl._core.restore_dataarray import restore_data_arrays_with_properties +from sympl._core.init_np_arrays import initialize_numpy_arrays_with_properties from ._components import ( PlotFunctionMonitor, NetCDFMonitor, RestartMonitor, ConstantTendencyComponent, ConstantDiagnosticComponent, RelaxationTendencyComponent, diff --git a/sympl/_components/basic.py b/sympl/_components/basic.py index 4f35e4f..f06648f 100644 --- a/sympl/_components/basic.py +++ b/sympl/_components/basic.py @@ -1,4 +1,4 @@ -from .._core.array import DataArray +from .._core.dataarray import DataArray from .._core.base_components import ImplicitTendencyComponent, TendencyComponent, DiagnosticComponent from .._core.units import unit_registry as ureg diff --git a/sympl/_components/netcdf.py b/sympl/_components/netcdf.py index 34245c5..29d7984 100644 --- a/sympl/_components/netcdf.py +++ b/sympl/_components/netcdf.py @@ -2,7 +2,7 @@ from .._core.exceptions import ( DependencyError, InvalidStateError) from .._core.units import from_unit_to_another -from .._core.array import DataArray +from .._core.dataarray import DataArray from .._core.util import same_list, datetime64_to_datetime import xarray as xr import os diff --git a/sympl/_components/plot.py b/sympl/_components/plot.py index b4969b2..21a6f24 100644 --- a/sympl/_components/plot.py +++ b/sympl/_components/plot.py @@ -1,6 +1,6 @@ from .._core.base_components import Monitor from .._core.exceptions import DependencyError -from .._core.array import DataArray +from .._core.dataarray import DataArray def copy_state(state): diff --git a/sympl/_components/timesteppers.py b/sympl/_components/timesteppers.py index 5c11561..5c84f6a 100644 --- a/sympl/_components/timesteppers.py +++ b/sympl/_components/timesteppers.py @@ -1,5 +1,5 @@ from .._core.prognosticstepper import PrognosticStepper -from .._core.array import DataArray +from .._core.dataarray import DataArray from .._core.state import copy_untouched_quantities, add, multiply diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index b81d935..8131048 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -1,5 +1,6 @@ import abc -from .state import get_numpy_arrays_with_properties, restore_data_arrays_with_properties +from .get_np_arrays import get_numpy_arrays_with_properties +from .restore_dataarray import restore_data_arrays_with_properties from .time import timedelta from .exceptions import ( InvalidPropertyDictError, ComponentExtraOutputError, @@ -611,14 +612,14 @@ def __call__(self, state, timestep): self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) if self.uses_tracers: - raw_state['tracers'] = self._tracer_packer.pack(raw_state) - for name in self._tracer_packer.tracer_names: - raw_state.pop(name) + raw_state['tracers'] = self._tracer_packer.pack(state) raw_state['time'] = state['time'] raw_diagnostics, raw_new_state = self.array_call(raw_state, timestep) if self.uses_tracers: - raw_new_state.update(self._tracer_packer.unpack(raw_new_state['tracers'])) - raw_new_state.pop('tracers') + new_state = self._tracer_packer.unpack( + raw_new_state.pop('tracers'), state) + else: + new_state = {} self._diagnostic_checker.check_diagnostics(raw_diagnostics) self._output_checker.check_outputs(raw_new_state) if self.tendencies_in_diagnostics: @@ -627,9 +628,9 @@ def __call__(self, state, timestep): diagnostics = restore_data_arrays_with_properties( raw_diagnostics, self.diagnostic_properties, state, self.input_properties) - new_state = restore_data_arrays_with_properties( + new_state.update(restore_data_arrays_with_properties( raw_new_state, self.output_properties, - state, self.input_properties) + state, self.input_properties)) return diagnostics, new_state def _insert_tendencies_to_diagnostics( @@ -707,7 +708,7 @@ def diagnostic_properties(self): name = None uses_tracers = False - tracer_tendency_time_unit = 's' + tracer_tendency_time_unit = 's^-1' def __str__(self): return ( @@ -837,26 +838,27 @@ def __call__(self, state): self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) if self.uses_tracers: - raw_state['tracers'] = self._tracer_packer.pack(raw_state) - for name in self._tracer_packer.tracer_names: - raw_state.pop(name) + raw_state['tracers'] = self._tracer_packer.pack(state) raw_state['time'] = state['time'] raw_tendencies, raw_diagnostics = self.array_call(raw_state) if self.uses_tracers: - raw_tendencies.update(self._tracer_packer.unpack(raw_tendencies['tracers'])) - raw_tendencies.pop('tracers') + out_tendencies = self._tracer_packer.unpack( + raw_tendencies.pop('tracers'), state, + multiply_unit=self.tracer_tendency_time_unit) + else: + out_tendencies = {} self._tendency_checker.check_tendencies(raw_tendencies) self._diagnostic_checker.check_diagnostics(raw_diagnostics) - tendencies = restore_data_arrays_with_properties( + out_tendencies.update(restore_data_arrays_with_properties( raw_tendencies, self.tendency_properties, - state, self.input_properties) + state, self.input_properties)) diagnostics = restore_data_arrays_with_properties( raw_diagnostics, self.diagnostic_properties, state, self.input_properties, ignore_names=self._added_diagnostic_names) if self.tendencies_in_diagnostics: - self._insert_tendencies_to_diagnostics(tendencies, diagnostics) - return tendencies, diagnostics + self._insert_tendencies_to_diagnostics(out_tendencies, diagnostics) + return out_tendencies, diagnostics def _insert_tendencies_to_diagnostics(self, tendencies, diagnostics): for name, value in tendencies.items(): @@ -931,7 +933,7 @@ def diagnostic_properties(self): name = None uses_tracers = False - tracer_tendency_time_unit = 's' + tracer_tendency_time_unit = 's^-1' def __str__(self): return ( @@ -1060,27 +1062,28 @@ def __call__(self, state, timestep): self._input_checker.check_inputs(state) raw_state = get_numpy_arrays_with_properties(state, self.input_properties) if self.uses_tracers: - raw_state['tracers'] = self._tracer_packer.pack(raw_state) - for name in self._tracer_packer.tracer_names: - raw_state.pop(name) + raw_state['tracers'] = self._tracer_packer.pack(state) raw_state['time'] = state['time'] raw_tendencies, raw_diagnostics = self.array_call(raw_state, timestep) if self.uses_tracers: - raw_tendencies.update(self._tracer_packer.unpack(raw_tendencies['tracers'])) - raw_tendencies.pop('tracers') + out_tendencies = self._tracer_packer.unpack( + raw_tendencies.pop('tracers'), state, + multiply_unit=self.tracer_tendency_time_unit) + else: + out_tendencies = {} self._tendency_checker.check_tendencies(raw_tendencies) self._diagnostic_checker.check_diagnostics(raw_diagnostics) - tendencies = restore_data_arrays_with_properties( + out_tendencies.update(restore_data_arrays_with_properties( raw_tendencies, self.tendency_properties, - state, self.input_properties) + state, self.input_properties)) diagnostics = restore_data_arrays_with_properties( raw_diagnostics, self.diagnostic_properties, state, self.input_properties, ignore_names=self._added_diagnostic_names) if self.tendencies_in_diagnostics: - self._insert_tendencies_to_diagnostics(tendencies, diagnostics) + self._insert_tendencies_to_diagnostics(out_tendencies, diagnostics) self._last_update_time = state['time'] - return tendencies, diagnostics + return out_tendencies, diagnostics def _insert_tendencies_to_diagnostics(self, tendencies, diagnostics): for name, value in tendencies.items(): diff --git a/sympl/_core/constants.py b/sympl/_core/constants.py index 4a16c9d..c5c5ec0 100644 --- a/sympl/_core/constants.py +++ b/sympl/_core/constants.py @@ -1,4 +1,4 @@ -from .array import DataArray +from .dataarray import DataArray from .units import is_valid_unit diff --git a/sympl/_core/array.py b/sympl/_core/dataarray.py similarity index 100% rename from sympl/_core/array.py rename to sympl/_core/dataarray.py index b063080..b6b3f84 100644 --- a/sympl/_core/array.py +++ b/sympl/_core/dataarray.py @@ -1,6 +1,6 @@ -from .units import data_array_to_units as to_units_function import xarray as xr from pint.errors import DimensionalityError +from .units import data_array_to_units as to_units_function class DataArray(xr.DataArray): diff --git a/sympl/_core/get_np_arrays.py b/sympl/_core/get_np_arrays.py new file mode 100644 index 0000000..9a84b57 --- /dev/null +++ b/sympl/_core/get_np_arrays.py @@ -0,0 +1,68 @@ +import numpy as np +from .exceptions import InvalidStateError +from .wildcard import get_wildcard_matches_and_dim_lengths, flatten_wildcard_dims + + +def get_numpy_arrays_with_properties(state, property_dictionary): + out_dict = {} + wildcard_names, dim_lengths = get_wildcard_matches_and_dim_lengths( + state, property_dictionary) + # Now we actually retrieve output arrays since we know the precise out dims + for name, properties in property_dictionary.items(): + ensure_quantity_has_units(state[name], name) + try: + quantity = state[name].to_units(properties['units']) + except ValueError: + raise InvalidStateError( + 'Could not convert quantity {} from units {} to units {}'.format( + name, state[name].attrs['units'], properties['units'] + ) + ) + out_dims = [] + out_dims.extend(properties['dims']) + has_wildcard = '*' in out_dims + if has_wildcard: + i_wildcard = out_dims.index('*') + out_dims[i_wildcard:i_wildcard+1] = wildcard_names + out_array = get_numpy_array( + quantity, out_dims=out_dims, dim_lengths=dim_lengths) + if has_wildcard: + out_array = flatten_wildcard_dims( + out_array, i_wildcard, i_wildcard + len(wildcard_names)) + if 'alias' in properties.keys(): + out_name = properties['alias'] + else: + out_name = name + out_dict[out_name] = out_array + return out_dict + + +def get_numpy_array(data_array, out_dims, dim_lengths): + """ + Gets a numpy array from the data_array with the desired out_dims, and a + dict of dim_lengths that will give the length of any missing dims in the + data_array. + """ + if len(data_array.values.shape) == 0 and len(out_dims) == 0: + return data_array.values # special case, 0-dimensional scalar array + else: + missing_dims = [dim for dim in out_dims if dim not in data_array.dims] + for dim in missing_dims: + data_array = data_array.expand_dims(dim) + numpy_array = data_array.transpose(*out_dims).values + if len(missing_dims) == 0: + out_array = numpy_array + else: # expand out missing dims which are currently length 1. + out_shape = [dim_lengths.get(name, 1) for name in out_dims] + if out_shape == list(numpy_array.shape): + out_array = numpy_array + else: + out_array = np.empty(out_shape, dtype=numpy_array.dtype) + out_array[:] = numpy_array + return out_array + + +def ensure_quantity_has_units(quantity, quantity_name): + if 'units' not in quantity.attrs: + raise InvalidStateError( + 'quantity {} is missing units attribute'.format(quantity_name)) diff --git a/sympl/_core/init_np_arrays.py b/sympl/_core/init_np_arrays.py new file mode 100644 index 0000000..c6298a2 --- /dev/null +++ b/sympl/_core/init_np_arrays.py @@ -0,0 +1,78 @@ +import numpy as np +from .tracers import get_tracer_names +from .exceptions import InvalidStateError +from .restore_dataarray import extract_output_dims_properties + + +def initialize_numpy_arrays_with_properties( + output_properties, raw_input_state, input_properties, tracer_dims=None, + prepend_tracers=()): + """ + Parameters + ---------- + output_properties : dict + A dictionary whose keys are quantity names and values are dictionaries + with properties for those quantities. The property "dims" must be + present for each quantity not also present in input_properties. + raw_input_state : dict + A state dictionary of numpy arrays that was used as input to a component + for which return arrays are being generated. + input_properties : dict + A dictionary whose keys are quantity names and values are dictionaries + with input properties for those quantities. The property "dims" must be + present, indicating the dimensions that the quantity was transformed to + when taken as input to a component. + + Returns + ------- + out_dict : dict + A dictionary whose keys are quantities and values are numpy arrays + corresponding to those quantities, with shapes determined from the + inputs to this function. + + Raises + ------ + InvalidPropertyDictError + When an output property is specified to have dims_like an input + property, but the arrays for the two properties have incompatible + shapes. + """ + dim_lengths = get_dim_lengths_from_raw_input(raw_input_state, input_properties) + dims_from_out_properties = extract_output_dims_properties( + output_properties, input_properties, []) + out_dict = {} + tracer_names = list(get_tracer_names()) + tracer_names.extend(entry[0] for entry in prepend_tracers) + for name, out_dims in dims_from_out_properties.items(): + if tracer_dims is None or name not in tracer_names: + out_shape = [] + for dim in out_dims: + out_shape.append(dim_lengths[dim]) + dtype = output_properties[name].get('dtype', np.float64) + out_dict[name] = np.zeros(out_shape, dtype=dtype) + if tracer_dims is not None: + out_shape = [] + dim_lengths['tracer'] = len(tracer_names) + for dim in tracer_dims: + out_shape.append(dim_lengths[dim]) + out_dict['tracers'] = np.zeros(out_shape, dtype=np.float64) + return out_dict + + +def get_dim_lengths_from_raw_input(raw_input, input_properties): + dim_lengths = {} + for name, properties in input_properties.items(): + if properties.get('tracer', False): + continue + if 'alias' in properties.keys(): + name = properties['alias'] + for i, dim_name in enumerate(properties['dims']): + if dim_name in dim_lengths: + if raw_input[name].shape[i] != dim_lengths[dim_name]: + raise InvalidStateError( + 'Dimension name {} has differing lengths on different ' + 'inputs'.format(dim_name) + ) + else: + dim_lengths[dim_name] = raw_input[name].shape[i] + return dim_lengths diff --git a/sympl/_core/restore_dataarray.py b/sympl/_core/restore_dataarray.py new file mode 100644 index 0000000..791fba1 --- /dev/null +++ b/sympl/_core/restore_dataarray.py @@ -0,0 +1,135 @@ +import numpy as np +from .exceptions import InvalidPropertyDictError +from .dataarray import DataArray +from .wildcard import ( + get_wildcard_matches_and_dim_lengths, fill_dims_wildcard, + expand_array_wildcard_dims +) + + +def ensure_values_are_arrays(array_dict): + for name, value in array_dict.items(): + if not isinstance(value, np.ndarray): + array_dict[name] = np.asarray(value) + + +def get_alias_or_name(name, output_properties, input_properties): + if 'alias' in output_properties[name].keys(): + raw_name = output_properties[name]['alias'] + elif name in input_properties.keys() and 'alias' in input_properties[name].keys(): + raw_name = input_properties[name]['alias'] + else: + raw_name = name + return raw_name + + +def check_array_shape(out_dims, raw_array, name, dim_lengths): + if len(out_dims) != len(raw_array.shape): + raise InvalidPropertyDictError( + 'Returned array for {} has shape {} ' + 'which is incompatible with dims {} in properties'.format( + name, raw_array.shape, out_dims)) + for dim, length in zip(out_dims, raw_array.shape): + if dim in dim_lengths.keys() and dim_lengths[dim] != length: + raise InvalidPropertyDictError( + 'Dimension {} of quantity {} has length {}, but ' + 'another quantity has length {}'.format( + dim, name, length, dim_lengths[dim]) + ) + + +def restore_data_arrays_with_properties( + raw_arrays, output_properties, input_state, input_properties, + ignore_names=None, ignore_missing=False): + """ + Parameters + ---------- + raw_arrays : dict + A dictionary whose keys are quantity names and values are numpy arrays + containing the data for those quantities. + output_properties : dict + A dictionary whose keys are quantity names and values are dictionaries + with properties for those quantities. The property "dims" must be + present for each quantity not also present in input_properties. All + other properties are included as attributes on the output DataArray + for that quantity, including "units" which is required. + input_state : dict + A state dictionary that was used as input to a component for which + DataArrays are being restored. + input_properties : dict + A dictionary whose keys are quantity names and values are dictionaries + with input properties for those quantities. The property "dims" must be + present, indicating the dimensions that the quantity was transformed to + when taken as input to a component. + ignore_names : iterable of str, optional + Names to ignore when encountered in output_properties, will not be + included in the returned dictionary. + ignore_missing : bool, optional + If True, ignore any values in output_properties not present in + raw_arrays rather than raising an exception. Default is False. + + Returns + ------- + out_dict : dict + A dictionary whose keys are quantities and values are DataArrays + corresponding to those quantities, with data, shapes and attributes + determined from the inputs to this function. + + Raises + ------ + InvalidPropertyDictError + When an output property is specified to have dims_like an input + property, but the arrays for the two properties have incompatible + shapes. + """ + raw_arrays = raw_arrays.copy() + if ignore_names is None: + ignore_names = [] + if ignore_missing: + ignore_names = set(output_properties.keys()).difference(raw_arrays.keys()).union(ignore_names) + wildcard_names, dim_lengths = get_wildcard_matches_and_dim_lengths( + input_state, input_properties) + ensure_values_are_arrays(raw_arrays) + dims_from_out_properties = extract_output_dims_properties( + output_properties, input_properties, ignore_names) + out_dict = {} + for name, out_dims in dims_from_out_properties.items(): + if name in ignore_names: + continue + raw_name = get_alias_or_name(name, output_properties, input_properties) + if '*' in out_dims: + for dim_name, length in zip(out_dims, raw_arrays[raw_name].shape): + if dim_name not in dim_lengths and dim_name != '*': + dim_lengths[dim_name] = length + out_dims_without_wildcard, target_shape = fill_dims_wildcard( + out_dims, dim_lengths, wildcard_names) + out_array = expand_array_wildcard_dims( + raw_arrays[raw_name], target_shape, name, out_dims) + else: + check_array_shape(out_dims, raw_arrays[raw_name], name, dim_lengths) + out_dims_without_wildcard = out_dims + out_array = raw_arrays[raw_name] + out_dict[name] = DataArray( + out_array, + dims=out_dims_without_wildcard, + attrs={'units': output_properties[name]['units']} + ) + return out_dict + + +def extract_output_dims_properties(output_properties, input_properties, ignore_names): + return_array = {} + for name, properties in output_properties.items(): + if name in ignore_names: + continue + elif 'dims' in properties.keys(): + return_array[name] = properties['dims'] + elif name not in input_properties.keys(): + raise InvalidPropertyDictError( + 'Output dims must be specified for {} in properties'.format(name)) + elif 'dims' not in input_properties[name].keys(): + raise InvalidPropertyDictError( + 'Input dims must be specified for {} in properties'.format(name)) + else: + return_array[name] = input_properties[name]['dims'] + return return_array diff --git a/sympl/_core/state.py b/sympl/_core/state.py index db3bead..04da25b 100644 --- a/sympl/_core/state.py +++ b/sympl/_core/state.py @@ -1,9 +1,3 @@ -from .exceptions import InvalidStateError, InvalidPropertyDictError -import numpy as np -from .array import DataArray -from .tracers import get_tracer_names - - def copy_untouched_quantities(old_state, new_state): for key in old_state.keys(): if key not in new_state: @@ -32,368 +26,3 @@ def multiply(scalar, state): if hasattr(out_state[key], 'attrs'): out_state[key].attrs = state[key].attrs return out_state - - -def get_wildcard_matches_and_dim_lengths(state, property_dictionary): - wildcard_names = [] - dim_lengths = {} - # Loop to get the set of names matching "*" (wildcard names) - for quantity_name, properties in property_dictionary.items(): - ensure_properties_have_dims_and_units(properties, quantity_name) - for dim_name, length in zip(state[quantity_name].dims, state[quantity_name].shape): - if dim_name not in dim_lengths.keys(): - dim_lengths[dim_name] = length - elif length != dim_lengths[dim_name]: - raise InvalidStateError( - 'Dimension {} conflicting lengths {} and {} in different ' - 'state quantities.'.format(dim_name, length, dim_lengths[dim_name])) - new_wildcard_names = [ - dim for dim in state[quantity_name].dims if dim not in properties['dims']] - if len(new_wildcard_names) > 0 and '*' not in properties['dims']: - raise InvalidStateError( - 'Quantity {} has unexpected dimensions {}.'.format( - quantity_name, new_wildcard_names)) - wildcard_names.extend( - [name for name in new_wildcard_names if name not in wildcard_names]) - if not any('dims' in p.keys() and '*' in p['dims'] for p in property_dictionary.values()): - wildcard_names = None # can't determine wildcard matches if there is no wildcard - else: - wildcard_names = tuple(wildcard_names) - return wildcard_names, dim_lengths - - -def get_numpy_arrays_with_properties(state, property_dictionary): - out_dict = {} - wildcard_names, dim_lengths = get_wildcard_matches_and_dim_lengths( - state, property_dictionary) - # Now we actually retrieve output arrays since we know the precise out dims - for name, properties in property_dictionary.items(): - ensure_quantity_has_units(state[name], name) - try: - quantity = state[name].to_units(properties['units']) - except ValueError: - raise InvalidStateError( - 'Could not convert quantity {} from units {} to units {}'.format( - name, state[name].attrs['units'], properties['units'] - ) - ) - out_dims = [] - out_dims.extend(properties['dims']) - has_wildcard = '*' in out_dims - if has_wildcard: - i_wildcard = out_dims.index('*') - out_dims[i_wildcard:i_wildcard+1] = wildcard_names - out_array = get_numpy_array( - quantity, out_dims=out_dims, dim_lengths=dim_lengths) - if has_wildcard: - out_array = flatten_wildcard_dims( - out_array, i_wildcard, i_wildcard + len(wildcard_names)) - if 'alias' in properties.keys(): - out_name = properties['alias'] - else: - out_name = name - out_dict[out_name] = out_array - return out_dict - - -def flatten_wildcard_dims(array, i_start, i_end): - if i_end > len(array.shape): - raise ValueError('i_end should be less than the number of axes in array') - elif i_start < 0: - raise ValueError('i_start should be greater than 0') - elif i_start > i_end: - raise ValueError('i_start should be less than or equal to i_end') - elif i_start == i_end: - # We need to insert a singleton dimension at i_start - target_shape = [] - target_shape.extend(array.shape) - target_shape.insert(i_start, 1) - else: - target_shape = [] - wildcard_length = 1 - for i, length in enumerate(array.shape): - if i_start <= i < i_end: - wildcard_length *= length - else: - target_shape.append(length) - if i == i_end - 1: - target_shape.append(wildcard_length) - return array.reshape(target_shape) - - -def get_numpy_array(data_array, out_dims, dim_lengths): - """ - Gets a numpy array from the data_array with the desired out_dims, and a - dict of dim_lengths that will give the length of any missing dims in the - data_array. - """ - if len(data_array.values.shape) == 0 and len(out_dims) == 0: - return data_array.values # special case, 0-dimensional scalar array - else: - missing_dims = [dim for dim in out_dims if dim not in data_array.dims] - for dim in missing_dims: - data_array = data_array.expand_dims(dim) - numpy_array = data_array.transpose(*out_dims).values - if len(missing_dims) == 0: - out_array = numpy_array - else: # expand out missing dims which are currently length 1. - out_shape = [dim_lengths.get(name, 1) for name in out_dims] - if out_shape == list(numpy_array.shape): - out_array = numpy_array - else: - out_array = np.empty(out_shape, dtype=numpy_array.dtype) - out_array[:] = numpy_array - return out_array - - -def initialize_numpy_arrays_with_properties( - output_properties, raw_input_state, input_properties, tracer_dims=None, - prepend_tracers=()): - """ - Parameters - ---------- - output_properties : dict - A dictionary whose keys are quantity names and values are dictionaries - with properties for those quantities. The property "dims" must be - present for each quantity not also present in input_properties. - raw_input_state : dict - A state dictionary of numpy arrays that was used as input to a component - for which return arrays are being generated. - input_properties : dict - A dictionary whose keys are quantity names and values are dictionaries - with input properties for those quantities. The property "dims" must be - present, indicating the dimensions that the quantity was transformed to - when taken as input to a component. - - Returns - ------- - out_dict : dict - A dictionary whose keys are quantities and values are numpy arrays - corresponding to those quantities, with shapes determined from the - inputs to this function. - - Raises - ------ - InvalidPropertyDictError - When an output property is specified to have dims_like an input - property, but the arrays for the two properties have incompatible - shapes. - """ - dim_lengths = get_dim_lengths_from_raw_input(raw_input_state, input_properties) - dims_from_out_properties = extract_output_dims_properties( - output_properties, input_properties, []) - out_dict = {} - tracer_names = list(get_tracer_names()) - tracer_names.extend(entry[0] for entry in prepend_tracers) - for name, out_dims in dims_from_out_properties.items(): - if tracer_dims is None or name not in tracer_names: - out_shape = [] - for dim in out_dims: - out_shape.append(dim_lengths[dim]) - dtype = output_properties[name].get('dtype', np.float64) - out_dict[name] = np.zeros(out_shape, dtype=dtype) - if tracer_dims is not None: - out_shape = [] - dim_lengths['tracer'] = len(tracer_names) - for dim in tracer_dims: - out_shape.append(dim_lengths[dim]) - out_dict['tracers'] = np.zeros(out_shape, dtype=np.float64) - return out_dict - - -def properties_include_tracers(input_properties): - for properties in input_properties.values(): - if properties.get('tracer', False): - return True - return False - - -def get_dim_lengths_from_raw_input(raw_input, input_properties): - dim_lengths = {} - for name, properties in input_properties.items(): - if properties.get('tracer', False): - continue - if 'alias' in properties.keys(): - name = properties['alias'] - for i, dim_name in enumerate(properties['dims']): - if dim_name in dim_lengths: - if raw_input[name].shape[i] != dim_lengths[dim_name]: - raise InvalidStateError( - 'Dimension name {} has differing lengths on different ' - 'inputs'.format(dim_name) - ) - else: - dim_lengths[dim_name] = raw_input[name].shape[i] - return dim_lengths - - -def ensure_values_are_arrays(array_dict): - for name, value in array_dict.items(): - if not isinstance(value, np.ndarray): - array_dict[name] = np.asarray(value) - - -def extract_output_dims_properties(output_properties, input_properties, ignore_names): - return_array = {} - for name, properties in output_properties.items(): - if name in ignore_names: - continue - elif 'dims' in properties.keys(): - return_array[name] = properties['dims'] - elif name not in input_properties.keys(): - raise InvalidPropertyDictError( - 'Output dims must be specified for {} in properties'.format(name)) - elif 'dims' not in input_properties[name].keys(): - raise InvalidPropertyDictError( - 'Input dims must be specified for {} in properties'.format(name)) - else: - return_array[name] = input_properties[name]['dims'] - return return_array - - -def fill_dims_wildcard( - out_dims, dim_lengths, wildcard_names, expand_wildcard=True): - i_wildcard = out_dims.index('*') - target_shape = [] - out_dims_without_wildcard = [] - for i, out_dim in enumerate(out_dims): - if i == i_wildcard and expand_wildcard: - target_shape.extend([dim_lengths[n] for n in wildcard_names]) - out_dims_without_wildcard.extend(wildcard_names) - elif i == i_wildcard and not expand_wildcard: - target_shape.append(np.product([dim_lengths[n] for n in wildcard_names])) - else: - target_shape.append(dim_lengths[out_dim]) - out_dims_without_wildcard.append(out_dim) - return out_dims_without_wildcard, target_shape - - -def expand_array_wildcard_dims(raw_array, target_shape, name, out_dims): - try: - out_array = np.reshape(raw_array, target_shape) - except ValueError: - raise InvalidPropertyDictError( - 'Failed to restore shape for output {} with raw shape {} ' - 'and target shape {}, are the output dims {} correct?'.format( - name, raw_array.shape, target_shape, - out_dims)) - return out_array - - -def get_alias_or_name(name, output_properties, input_properties): - if 'alias' in output_properties[name].keys(): - raw_name = output_properties[name]['alias'] - elif name in input_properties.keys() and 'alias' in input_properties[name].keys(): - raw_name = input_properties[name]['alias'] - else: - raw_name = name - return raw_name - - -def check_array_shape(out_dims, raw_array, name, dim_lengths): - if len(out_dims) != len(raw_array.shape): - raise InvalidPropertyDictError( - 'Returned array for {} has shape {} ' - 'which is incompatible with dims {} in properties'.format( - name, raw_array.shape, out_dims)) - for dim, length in zip(out_dims, raw_array.shape): - if dim in dim_lengths.keys() and dim_lengths[dim] != length: - raise InvalidPropertyDictError( - 'Dimension {} of quantity {} has length {}, but ' - 'another quantity has length {}'.format( - dim, name, length, dim_lengths[dim]) - ) - - -def restore_data_arrays_with_properties( - raw_arrays, output_properties, input_state, input_properties, - ignore_names=None, ignore_missing=False): - """ - Parameters - ---------- - raw_arrays : dict - A dictionary whose keys are quantity names and values are numpy arrays - containing the data for those quantities. - output_properties : dict - A dictionary whose keys are quantity names and values are dictionaries - with properties for those quantities. The property "dims" must be - present for each quantity not also present in input_properties. All - other properties are included as attributes on the output DataArray - for that quantity, including "units" which is required. - input_state : dict - A state dictionary that was used as input to a component for which - DataArrays are being restored. - input_properties : dict - A dictionary whose keys are quantity names and values are dictionaries - with input properties for those quantities. The property "dims" must be - present, indicating the dimensions that the quantity was transformed to - when taken as input to a component. - ignore_names : iterable of str, optional - Names to ignore when encountered in output_properties, will not be - included in the returned dictionary. - ignore_missing : bool, optional - If True, ignore any values in output_properties not present in - raw_arrays rather than raising an exception. Default is False. - - Returns - ------- - out_dict : dict - A dictionary whose keys are quantities and values are DataArrays - corresponding to those quantities, with data, shapes and attributes - determined from the inputs to this function. - - Raises - ------ - InvalidPropertyDictError - When an output property is specified to have dims_like an input - property, but the arrays for the two properties have incompatible - shapes. - """ - raw_arrays = raw_arrays.copy() - if ignore_names is None: - ignore_names = [] - if ignore_missing: - ignore_names = set(output_properties.keys()).difference(raw_arrays.keys()).union(ignore_names) - wildcard_names, dim_lengths = get_wildcard_matches_and_dim_lengths( - input_state, input_properties) - ensure_values_are_arrays(raw_arrays) - dims_from_out_properties = extract_output_dims_properties( - output_properties, input_properties, ignore_names) - out_dict = {} - for name, out_dims in dims_from_out_properties.items(): - if name in ignore_names: - continue - raw_name = get_alias_or_name(name, output_properties, input_properties) - if '*' in out_dims: - for dim_name, length in zip(out_dims, raw_arrays[raw_name].shape): - if dim_name not in dim_lengths and dim_name != '*': - dim_lengths[dim_name] = length - out_dims_without_wildcard, target_shape = fill_dims_wildcard( - out_dims, dim_lengths, wildcard_names) - out_array = expand_array_wildcard_dims( - raw_arrays[raw_name], target_shape, name, out_dims) - else: - check_array_shape(out_dims, raw_arrays[raw_name], name, dim_lengths) - out_dims_without_wildcard = out_dims - out_array = raw_arrays[raw_name] - out_dict[name] = DataArray( - out_array, - dims=out_dims_without_wildcard, - attrs={'units': output_properties[name]['units']} - ) - return out_dict - - -def ensure_properties_have_dims_and_units(properties, quantity_name): - if 'dims' not in properties: - raise InvalidPropertyDictError( - 'dims not specified for quantity {}'.format(quantity_name)) - if 'units' not in properties: - raise InvalidPropertyDictError( - 'units not specified for quantity {}'.format(quantity_name)) - - -def ensure_quantity_has_units(quantity, quantity_name): - if 'units' not in quantity.attrs: - raise InvalidStateError( - 'quantity {} is missing units attribute'.format(quantity_name)) diff --git a/sympl/_core/tracers.py b/sympl/_core/tracers.py index 6b59930..ee2150e 100644 --- a/sympl/_core/tracers.py +++ b/sympl/_core/tracers.py @@ -1,6 +1,8 @@ from .exceptions import InvalidPropertyDictError import numpy as np from .units import units_are_same +from .get_np_arrays import get_numpy_arrays_with_properties +from .restore_dataarray import restore_data_arrays_with_properties _tracer_unit_dict = {} _tracer_names = [] @@ -39,7 +41,7 @@ def register_tracer(name, units): _tracer_unit_dict[name] = units _tracer_names.append(name) for packer in _packers: - packer.insert_tracer_to_properties(name, units) + packer.ensure_tracer_not_in_outputs(name, units) def get_tracer_unit_dict(): @@ -65,6 +67,27 @@ def get_tracer_names(): return tuple(_tracer_names) +def get_tracer_properties(prepend_tracers, tracer_dims): + """ + + Args: + prepend_tracers (list of tuple): Pairs of (name, units) describing + tracers that are to be included in addition to any registered + tracers. + + Returns: + input_properties (dict): A properties dictionary for registered and + additional tracers. + """ + tracer_units = {} + tracer_units.update(get_tracer_unit_dict()) + tracer_units.update(dict(prepend_tracers)) + tracer_properties = {} + for name, units in tracer_units.items(): + tracer_properties[name] = {'units': units, 'dims': tracer_dims} + return tracer_properties + + def get_quantity_dims(tracer_dims): if 'tracer' not in tracer_dims: raise ValueError("Tracer dims must include a dimension named 'tracer'") @@ -88,34 +111,18 @@ def __init__(self, component, tracer_dims, prepend_tracers=None): 'type {}'.format(component.__class__.__name__)) for name, units in self._prepend_tracers: if name not in _tracer_unit_dict.keys(): - self.insert_tracer_to_properties(name, units) + self.ensure_tracer_not_in_outputs(name, units) for name, units in _tracer_unit_dict.items(): - self.insert_tracer_to_properties(name, units) + self.ensure_tracer_not_in_outputs(name, units) _packers.add(self) - def insert_tracer_to_properties(self, name, units): - self._insert_tracer_to_input_properties(name, units) + def ensure_tracer_not_in_outputs(self, name, units): if hasattr(self.component, 'tendency_properties'): - self._insert_tracer_to_tendency_properties(name, units) + self._ensure_tracer_not_in_tendency_properties(name, units) elif hasattr(self.component, 'output_properties'): - self._insert_tracer_to_output_properties(name, units) + self.ensure_tracer_not_in_output_properties(name, units) - def _insert_tracer_to_input_properties(self, name, units): - if name in self.component.input_properties.keys(): - raise InvalidPropertyDictError( - 'Attempted to insert {} as tracer to component of type {} but ' - 'it already has that quantity defined as an input.'.format( - name, self.component.__class__.__name__ - ) - ) - if name not in self.component.input_properties: - self.component.input_properties[name] = { - 'dims': self._tracer_quantity_dims, - 'units': units, - 'tracer': True, - } - - def _insert_tracer_to_output_properties(self, name, units): + def ensure_tracer_not_in_output_properties(self, name, units): if name in self.component.output_properties.keys(): raise InvalidPropertyDictError( 'Attempted to insert {} as tracer to component of type {} but ' @@ -123,15 +130,8 @@ def _insert_tracer_to_output_properties(self, name, units): name, self.component.__class__.__name__ ) ) - if name not in self.component.output_properties: - self.component.output_properties[name] = { - 'dims': self._tracer_quantity_dims, - 'units': units, - 'tracer': True, - } - - def _insert_tracer_to_tendency_properties(self, name, units): - time_unit = getattr(self.component, 'tracer_tendency_time_unit', 's') + + def _ensure_tracer_not_in_tendency_properties(self, name, units): if name in self.component.tendency_properties.keys(): raise InvalidPropertyDictError( 'Attempted to insert {} as tracer to component of type {} but ' @@ -140,15 +140,6 @@ def _insert_tracer_to_tendency_properties(self, name, units): name, self.component.__class__.__name__ ) ) - if name not in self.component.tendency_properties: - self.component.tendency_properties[name] = { - 'dims': self._tracer_quantity_dims, - 'units': '{} {}^-1'.format(units, time_unit), - 'tracer': True, - } - - def is_tracer(self, tracer_name): - return self.component.input_properties.get(tracer_name, {}).get('tracer', False) def _ensure_tracer_quantity_dims(self, dims): if tuple(self._tracer_quantity_dims) != tuple(dims): @@ -172,7 +163,7 @@ def tracer_names(self): for name, units in self._prepend_tracers: return_list.append(name) for name in _tracer_names: - if name not in return_list and self.is_tracer(name): + if name not in return_list: return_list.append(name) return tuple(return_list) @@ -180,7 +171,20 @@ def tracer_names(self): def _tracer_index(self): return self._tracer_dims.index('tracer') - def pack(self, raw_state): + def pack(self, state): + """ + + Args: + state (dict): A state dictionary. + + Returns: + tracer_array (ndarray): An array containing the tracer data, with + dimensions as specified by tracer_dims on initializing this + object. + """ + tracer_properties = get_tracer_properties( + self._prepend_tracers, self._tracer_quantity_dims) + raw_state = get_numpy_arrays_with_properties(state, tracer_properties) if len(self.tracer_names) == 0: shape = [0 for dim in self._tracer_dims] else: @@ -193,10 +197,37 @@ def pack(self, raw_state): array[tracer_slice] = raw_state[name] return array - def unpack(self, tracer_array): - return_state = {} + def unpack(self, tracer_array, input_state, multiply_unit=''): + """ + + Args: + tracer_array (ndarray): An array containing tracer values, with + dimensions as specified by tracer_dims on initializing this + object. + input_state (dict): A state dictionary from which the tracer array + was originally packed. + multiply_unit (str, optional): A unit string which should be multiplied to + the units of each output in the returned DataArrays, for example + to represent the units over which a time difference was taken. + + Returns: + tracer_dict (dict): A dictionary whose keys are tracer names and + values are DataArrays containing the values of each + tracer. + """ + tracer_properties = get_tracer_properties( + self._prepend_tracers, self._tracer_quantity_dims) + raw_state = {} for i, name in enumerate(self.tracer_names): tracer_slice = [slice(0, d) for d in tracer_array.shape] tracer_slice[self._tracer_index] = i - return_state[name] = tracer_array[tracer_slice] + raw_state[name] = tracer_array[tracer_slice] + out_properties = {} + for name, properties in tracer_properties.items(): + out_properties[name] = properties.copy() + if multiply_unit is not '': + out_properties[name]['units'] = '{} {}'.format( + out_properties[name]['units'], multiply_unit) + return_state = restore_data_arrays_with_properties( + raw_state, out_properties, input_state, tracer_properties) return return_state diff --git a/sympl/_core/util.py b/sympl/_core/util.py index 5bf6f2f..a26fcc8 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -1,7 +1,7 @@ from datetime import datetime import numpy as np from .units import units_are_compatible -from .array import DataArray +from .dataarray import DataArray from .exceptions import ( SharedKeyError, InvalidPropertyDictError) diff --git a/sympl/_core/wildcard.py b/sympl/_core/wildcard.py new file mode 100644 index 0000000..9628a84 --- /dev/null +++ b/sympl/_core/wildcard.py @@ -0,0 +1,93 @@ +import numpy as np +from .exceptions import InvalidStateError, InvalidPropertyDictError + + +def get_wildcard_matches_and_dim_lengths(state, property_dictionary): + wildcard_names = [] + dim_lengths = {} + # Loop to get the set of names matching "*" (wildcard names) + for quantity_name, properties in property_dictionary.items(): + ensure_properties_have_dims_and_units(properties, quantity_name) + for dim_name, length in zip(state[quantity_name].dims, state[quantity_name].shape): + if dim_name not in dim_lengths.keys(): + dim_lengths[dim_name] = length + elif length != dim_lengths[dim_name]: + raise InvalidStateError( + 'Dimension {} conflicting lengths {} and {} in different ' + 'state quantities.'.format(dim_name, length, dim_lengths[dim_name])) + new_wildcard_names = [ + dim for dim in state[quantity_name].dims if dim not in properties['dims']] + if len(new_wildcard_names) > 0 and '*' not in properties['dims']: + raise InvalidStateError( + 'Quantity {} has unexpected dimensions {}.'.format( + quantity_name, new_wildcard_names)) + wildcard_names.extend( + [name for name in new_wildcard_names if name not in wildcard_names]) + if not any('dims' in p.keys() and '*' in p['dims'] for p in property_dictionary.values()): + wildcard_names = None # can't determine wildcard matches if there is no wildcard + else: + wildcard_names = tuple(wildcard_names) + return wildcard_names, dim_lengths + + +def flatten_wildcard_dims(array, i_start, i_end): + if i_end > len(array.shape): + raise ValueError('i_end should be less than the number of axes in array') + elif i_start < 0: + raise ValueError('i_start should be greater than 0') + elif i_start > i_end: + raise ValueError('i_start should be less than or equal to i_end') + elif i_start == i_end: + # We need to insert a singleton dimension at i_start + target_shape = [] + target_shape.extend(array.shape) + target_shape.insert(i_start, 1) + else: + target_shape = [] + wildcard_length = 1 + for i, length in enumerate(array.shape): + if i_start <= i < i_end: + wildcard_length *= length + else: + target_shape.append(length) + if i == i_end - 1: + target_shape.append(wildcard_length) + return array.reshape(target_shape) + + +def fill_dims_wildcard( + out_dims, dim_lengths, wildcard_names, expand_wildcard=True): + i_wildcard = out_dims.index('*') + target_shape = [] + out_dims_without_wildcard = [] + for i, out_dim in enumerate(out_dims): + if i == i_wildcard and expand_wildcard: + target_shape.extend([dim_lengths[n] for n in wildcard_names]) + out_dims_without_wildcard.extend(wildcard_names) + elif i == i_wildcard and not expand_wildcard: + target_shape.append(np.product([dim_lengths[n] for n in wildcard_names])) + else: + target_shape.append(dim_lengths[out_dim]) + out_dims_without_wildcard.append(out_dim) + return out_dims_without_wildcard, target_shape + + +def expand_array_wildcard_dims(raw_array, target_shape, name, out_dims): + try: + out_array = np.reshape(raw_array, target_shape) + except ValueError: + raise InvalidPropertyDictError( + 'Failed to restore shape for output {} with raw shape {} ' + 'and target shape {}, are the output dims {} correct?'.format( + name, raw_array.shape, target_shape, + out_dims)) + return out_array + + +def ensure_properties_have_dims_and_units(properties, quantity_name): + if 'dims' not in properties: + raise InvalidPropertyDictError( + 'dims not specified for quantity {}'.format(quantity_name)) + if 'units' not in properties: + raise InvalidPropertyDictError( + 'units not specified for quantity {}'.format(quantity_name)) diff --git a/tests/test_tracers.py b/tests/test_tracers.py index ee6fe3a..93f3a3f 100644 --- a/tests/test_tracers.py +++ b/tests/test_tracers.py @@ -1,7 +1,7 @@ from sympl._core.tracers import TracerPacker, reset_tracers, reset_packers from sympl import ( TendencyComponent, Stepper, DiagnosticComponent, ImplicitTendencyComponent, register_tracer, - get_tracer_unit_dict, units_are_compatible, DataArray + get_tracer_unit_dict, units_are_compatible, DataArray, InvalidPropertyDictError ) import unittest import numpy as np @@ -16,9 +16,9 @@ class MockTendencyComponent(TendencyComponent): tendency_properties = None def __init__(self, **kwargs): - self.input_properties = {} - self.diagnostic_properties = {} - self.tendency_properties = {} + self.input_properties = kwargs.pop('input_properties', {}) + self.diagnostic_properties = kwargs.pop('diagnostic_properties', {}) + self.tendency_properties = kwargs.pop('tendency_properties', {}) self.diagnostic_output = {} self.tendency_output = {} self.times_called = 0 @@ -44,9 +44,9 @@ def __init__(self, **kwargs): prepend_tracers = kwargs.pop('prepend_tracers', None) if prepend_tracers is not None: self.prepend_tracers = prepend_tracers - self.input_properties = {} - self.diagnostic_properties = {} - self.tendency_properties = {} + self.input_properties = kwargs.pop('input_properties', {}) + self.diagnostic_properties = kwargs.pop('diagnostic_properties', {}) + self.tendency_properties = kwargs.pop('tendency_properties', {}) self.diagnostic_output = {} self.times_called = 0 self.state_given = None @@ -68,9 +68,9 @@ class MockImplicitTendencyComponent(ImplicitTendencyComponent): tendency_properties = None def __init__( self, **kwargs): - self.input_properties = {} - self.diagnostic_properties = {} - self.tendency_properties = {} + self.input_properties = kwargs.pop('input_properties', {}) + self.diagnostic_properties = kwargs.pop('diagnostic_properties', {}) + self.tendency_properties = kwargs.pop('tendency_properties', {}) self.diagnostic_output = {} self.tendency_output = {} self.times_called = 0 @@ -98,9 +98,9 @@ def __init__(self, **kwargs): prepend_tracers = kwargs.pop('prepend_tracers', None) if prepend_tracers is not None: self.prepend_tracers = prepend_tracers - self.input_properties = {} - self.diagnostic_properties = {} - self.tendency_properties = {} + self.input_properties = kwargs.pop('input_properties', {}) + self.diagnostic_properties = kwargs.pop('diagnostic_properties', {}) + self.tendency_properties = kwargs.pop('tendency_properties', {}) self.diagnostic_output = {} self.times_called = 0 self.state_given = None @@ -123,8 +123,8 @@ class MockDiagnosticComponent(DiagnosticComponent): diagnostic_properties = None def __init__(self, **kwargs): - self.input_properties = {} - self.diagnostic_properties = {} + self.input_properties = kwargs.pop('input_properties', {}) + self.diagnostic_properties = kwargs.pop('diagnostic_properties', {}) self.diagnostic_output = {} self.times_called = 0 self.state_given = None @@ -143,9 +143,9 @@ class MockStepper(Stepper): output_properties = None def __init__(self, **kwargs): - self.input_properties = {} - self.diagnostic_properties = {} - self.output_properties = {} + self.input_properties = kwargs.pop('input_properties', {}) + self.diagnostic_properties = kwargs.pop('diagnostic_properties', {}) + self.output_properties = kwargs.pop('output_properties', {}) self.diagnostic_output = {} self.state_output = {} self.times_called = 0 @@ -173,9 +173,9 @@ def __init__(self, **kwargs): prepend_tracers = kwargs.pop('prepend_tracers', None) if prepend_tracers is not None: self.prepend_tracers = prepend_tracers - self.input_properties = {} - self.diagnostic_properties = {} - self.output_properties = {} + self.input_properties = kwargs.pop('input_properties', {}) + self.diagnostic_properties = kwargs.pop('diagnostic_properties', {}) + self.output_properties = kwargs.pop('output_properties', {}) self.diagnostic_output = {} self.state_output = {} self.times_called = 0 @@ -255,80 +255,86 @@ def test_packs_no_tracers(self): def test_unpacks_no_tracers(self): dims = ['tracer', '*'] packer = TracerPacker(self.component, dims) - unpacked = packer.unpack({}) + unpacked = packer.unpack({}, {}) assert isinstance(unpacked, dict) assert len(unpacked) == 0 - def test_unpacks_no_tracers_with_arrays_input(self): + def test_packs_one_tracer(self): + np.random.seed(0) dims = ['tracer', '*'] + register_tracer('tracer1', 'g/m^3') + state = {'tracer1': DataArray(np.random.randn(5), dims=['dim1'], attrs={'units': 'g/m^3'})} packer = TracerPacker(self.component, dims) - unpacked = packer.unpack({'air_temperature': np.zeros((5,))}) - assert isinstance(unpacked, dict) - assert len(unpacked) == 0 + packed = packer.pack(state) + assert isinstance(packed, np.ndarray) + assert packed.shape == (1, 5) + assert np.all(packed[0, :] == state['tracer1'].values) - def test_packs_one_tracer(self): + def test_packs_one_tracer_converts_units(self): np.random.seed(0) dims = ['tracer', '*'] - register_tracer('tracer1', 'g/m^3') - raw_state = {'tracer1': np.random.randn(5)} + register_tracer('tracer1', 'kg/m^3') + state = {'tracer1': DataArray(np.random.randn(5), dims=['dim1'], attrs={'units': 'g/m^3'})} packer = TracerPacker(self.component, dims) - packed = packer.pack(raw_state) + packed = packer.pack(state) assert isinstance(packed, np.ndarray) assert packed.shape == (1, 5) - assert np.all(packed[0, :] == raw_state['tracer1']) + assert np.all(packed[0, :] == state['tracer1'].values * 1e-3) def test_packs_one_3d_tracer(self): np.random.seed(0) dims = ['tracer', 'latitude', 'longitude', 'mid_levels'] register_tracer('tracer1', 'g/m^3') - raw_state = {'tracer1': np.random.randn(2, 3, 4)} + state = { + 'tracer1': DataArray( + np.random.randn(2, 3, 4), + dims=['latitude', 'longitude', 'mid_levels'], + attrs={'units': 'g/m^3'} + ) + } packer = TracerPacker(self.component, dims) - packed = packer.pack(raw_state) + packed = packer.pack(state) assert isinstance(packed, np.ndarray) assert packed.shape == (1, 2, 3, 4) - assert np.all(packed[0, :, :, :] == raw_state['tracer1']) + assert np.all(packed[0, :, :, :] == state['tracer1'].values) - def test_packs_updates_input_properties(self): + def test_packer_does_not_change_input_properties(self): np.random.seed(0) dims = ['tracer', '*'] register_tracer('tracer1', 'g/m^3') packer = TracerPacker(self.component, dims) - assert 'tracer1' in self.component.input_properties - assert tuple(self.component.input_properties['tracer1']['dims']) == ('*',) - assert self.component.input_properties['tracer1']['units'] == 'g/m^3' - assert len(self.component.input_properties) == 1 + assert len(self.component.input_properties) == 0 - def test_packs_updates_input_properties_after_init(self): + def test_packer_does_not_change_input_properties_after_init(self): np.random.seed(0) dims = ['tracer', '*'] packer = TracerPacker(self.component, dims) assert len(self.component.input_properties) == 0 register_tracer('tracer1', 'g/m^3') - assert 'tracer1' in self.component.input_properties - assert tuple( - self.component.input_properties['tracer1']['dims']) == ('*',) - assert self.component.input_properties['tracer1']['units'] == 'g/m^3' - assert len(self.component.input_properties) == 1 + assert len(self.component.input_properties) == 0 def test_packs_one_tracer_registered_after_init(self): np.random.seed(0) dims = ['tracer', '*'] - raw_state = {'tracer1': np.random.randn(5)} + state = {'tracer1': DataArray(np.random.randn(5), dims=['dim1'], attrs={'units': 'g/m^3'})} packer = TracerPacker(self.component, dims) register_tracer('tracer1', 'g/m^3') - packed = packer.pack(raw_state) + packed = packer.pack(state) assert isinstance(packed, np.ndarray) assert packed.shape == (1, 5) - assert np.all(packed[0, :] == raw_state['tracer1']) + assert np.all(packed[0, :] == state['tracer1'].values) def test_packs_two_tracers(self): np.random.seed(0) dims = ['tracer', '*'] register_tracer('tracer1', 'g/m^3') register_tracer('tracer2', 'kg') - raw_state = {'tracer1': np.random.randn(5), 'tracer2': np.random.randn(5)} + state = { + 'tracer1': DataArray(np.random.randn(5), dims=['dim1'], attrs={'units': 'g/m^3'}), + 'tracer2': DataArray(np.random.randn(5), dims=['dim1'], attrs={'units': 'kg'}) + } packer = TracerPacker(self.component, dims) - packed = packer.pack(raw_state) + packed = packer.pack(state) assert isinstance(packed, np.ndarray) assert packed.shape == (2, 5) @@ -338,18 +344,18 @@ def test_packs_three_tracers_in_order_registered(self): register_tracer('tracer1', 'g/m^3') register_tracer('tracer2', 'kg'), register_tracer('tracer3', 'kg/m^3') - raw_state = { - 'tracer1': np.random.randn(5), - 'tracer2': np.random.randn(5), - 'tracer3': np.random.randn(5), + state = { + 'tracer1': DataArray(np.random.randn(5), dims=['dim1'], attrs={'units': 'g/m^3'}), + 'tracer2': DataArray(np.random.randn(5), dims=['dim1'], attrs={'units': 'kg'}), + 'tracer3': DataArray(np.random.randn(5), dims=['dim1'], attrs={'units': 'kg/m^3'}), } packer = TracerPacker(self.component, dims) - packed = packer.pack(raw_state) + packed = packer.pack(state) assert isinstance(packed, np.ndarray) assert packed.shape == (3, 5) - assert np.all(packed[0, :] == raw_state['tracer1']) - assert np.all(packed[1, :] == raw_state['tracer2']) - assert np.all(packed[2, :] == raw_state['tracer3']) + assert np.all(packed[0, :] == state['tracer1'].values) + assert np.all(packed[1, :] == state['tracer2'].values) + assert np.all(packed[2, :] == state['tracer3'].values) def test_unpacks_three_tracers_in_order_registered(self): np.random.seed(0) @@ -357,19 +363,43 @@ def test_unpacks_three_tracers_in_order_registered(self): register_tracer('tracer1', 'g/m^3') register_tracer('tracer2', 'kg'), register_tracer('tracer3', 'kg/m^3') - raw_state = { - 'tracer1': np.random.randn(5), - 'tracer2': np.random.randn(5), - 'tracer3': np.random.randn(5), + state = { + 'tracer1': DataArray(np.random.randn(5), dims=['dim1'], attrs={'units': 'g/m^3'}), + 'tracer2': DataArray(np.random.randn(5), dims=['dim1'], attrs={'units': 'kg'}), + 'tracer3': DataArray(np.random.randn(5), dims=['dim1'], attrs={'units': 'kg/m^3'}), } packer = TracerPacker(self.component, dims) - packed = packer.pack(raw_state) - unpacked = packer.unpack(packed) + packed = packer.pack(state) + unpacked = packer.unpack(packed, state) assert isinstance(unpacked, dict) assert len(unpacked) == 3 - assert np.all(unpacked['tracer1'] == raw_state['tracer1']) - assert np.all(unpacked['tracer2'] == raw_state['tracer2']) - assert np.all(unpacked['tracer3'] == raw_state['tracer3']) + assert np.all(unpacked['tracer1'] == state['tracer1']) + assert np.all(unpacked['tracer2'] == state['tracer2']) + assert np.all(unpacked['tracer3'] == state['tracer3']) + + def test_packer_allows_overlap_input_registered_after_init(self): + self.component = self.component.__class__( + input_properties={ + 'name': { + 'units': 'm', + 'dims': ['*'], + } + } + ) + packer = TracerPacker(self.component, ['tracer', '*']) + register_tracer('name', 'm') + + def test_packer_allows_overlap_input_registered_before_init(self): + self.component = self.component.__class__( + input_properties={ + 'name': { + 'units': 'm', + 'dims': ['*'], + } + } + ) + register_tracer('name', 'm') + packer = TracerPacker(self.component, ['tracer', '*']) class PrognosticTracerPackerTests(TracerPackerBase, unittest.TestCase): @@ -382,27 +412,47 @@ def tearDown(self): self.component = None super(PrognosticTracerPackerTests, self).tearDown() - def test_packs_updates_tendency_properties(self): + def test_packer_does_not_change_tendency_properties(self): np.random.seed(0) dims = ['tracer', '*'] register_tracer('tracer1', 'g/m^3') packer = TracerPacker(self.component, dims) - assert 'tracer1' in self.component.tendency_properties - assert tuple(self.component.tendency_properties['tracer1']['dims']) == ('*',) - assert units_are_compatible(self.component.tendency_properties['tracer1']['units'], 'g/m^3 s^-1') - assert len(self.component.tendency_properties) == 1 + assert 'tracer1' not in self.component.tendency_properties + assert len(self.component.tendency_properties) == 0 - def test_packs_updates_tendency_properties_after_init(self): + def test_packer_does_not_change_tendency_properties_after_init(self): np.random.seed(0) dims = ['tracer', '*'] packer = TracerPacker(self.component, dims) assert len(self.component.tendency_properties) == 0 register_tracer('tracer1', 'g/m^3') - assert 'tracer1' in self.component.tendency_properties - assert tuple( - self.component.tendency_properties['tracer1']['dims']) == ('*',) - assert units_are_compatible(self.component.tendency_properties['tracer1']['units'], 'g/m^3 s^-1') - assert len(self.component.tendency_properties) == 1 + assert len(self.component.tendency_properties) == 0 + + def test_packer_wont_overwrite_tendency_registered_after_init(self): + self.component = MockTendencyComponent( + tendency_properties={ + 'name': { + 'units': 'm', + 'dims': ['*'], + } + } + ) + packer = TracerPacker(self.component, ['tracer', '*']) + with self.assertRaises(InvalidPropertyDictError): + register_tracer('name', 'm') + + def test_packer_wont_overwrite_tendency_registered_before_init(self): + self.component = MockTendencyComponent( + tendency_properties={ + 'name': { + 'units': 'm', + 'dims': ['*'], + } + } + ) + register_tracer('name', 'm') + with self.assertRaises(InvalidPropertyDictError): + packer = TracerPacker(self.component, ['tracer', '*']) class ImplicitPrognosticTracerPackerTests(PrognosticTracerPackerTests): @@ -426,27 +476,46 @@ def tearDown(self): self.component = None super(ImplicitTracerPackerTests, self).tearDown() - def test_packs_updates_output_properties(self): + def test_packer_does_not_change_output_properties(self): np.random.seed(0) dims = ['tracer', '*'] register_tracer('tracer1', 'g/m^3') packer = TracerPacker(self.component, dims) - assert 'tracer1' in self.component.output_properties - assert tuple(self.component.output_properties['tracer1']['dims']) == ('*',) - assert self.component.output_properties['tracer1']['units'] == 'g/m^3' - assert len(self.component.output_properties) == 1 + assert len(self.component.output_properties) == 0 - def test_packs_updates_output_properties_after_init(self): + def test_packer_does_not_change_output_properties_after_init(self): np.random.seed(0) dims = ['tracer', '*'] packer = TracerPacker(self.component, dims) assert len(self.component.output_properties) == 0 register_tracer('tracer1', 'g/m^3') - assert 'tracer1' in self.component.output_properties - assert tuple( - self.component.output_properties['tracer1']['dims']) == ('*',) - assert self.component.output_properties['tracer1']['units'] == 'g/m^3' - assert len(self.component.output_properties) == 1 + assert len(self.component.output_properties) == 0 + + def test_packer_wont_overwrite_output_registered_after_init(self): + self.component = MockStepper( + output_properties={ + 'name': { + 'units': 'm', + 'dims': ['*'], + } + } + ) + packer = TracerPacker(self.component, ['tracer', '*']) + with self.assertRaises(InvalidPropertyDictError): + register_tracer('name', 'm') + + def test_packer_wont_overwrite_output_registered_before_init(self): + self.component = MockStepper( + output_properties={ + 'name': { + 'units': 'm', + 'dims': ['*'], + } + } + ) + register_tracer('name', 'm') + with self.assertRaises(InvalidPropertyDictError): + packer = TracerPacker(self.component, ['tracer', '*']) class DiagnosticTracerPackerTests(unittest.TestCase): @@ -639,13 +708,10 @@ def test_restores_one_3d_tracer(self): assert unpacked['tracer1'].shape == input_state['tracer1'].shape assert np.all(unpacked['tracer1'].values == input_state['tracer1'].values) - def test_updates_input_properties(self): + def test_does_not_change_input_properties(self): np.random.seed(0) register_tracer('tracer1', 'g/m^3') - assert 'tracer1' in self.component.input_properties - assert tuple(self.component.input_properties['tracer1']['dims']) == ('*',) - assert self.component.input_properties['tracer1']['units'] == 'g/m^3' - assert len(self.component.input_properties) == 1 + assert 'tracer1' not in self.component.input_properties def test_packs_two_tracers(self): np.random.seed(0) From 5aeab9eaac09b1b6a349b70dd355b5a112be4a9e Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Jul 2018 21:43:36 -0700 Subject: [PATCH 86/98] Moved around property code to avoid circular dependency --- sympl/__init__.py | 9 +-- sympl/_core/combine_properties.py | 101 ++++++++++++++++++++++++++++++ sympl/_core/composite.py | 4 +- sympl/_core/prognosticstepper.py | 2 +- sympl/_core/tracers.py | 8 ++- sympl/_core/util.py | 101 +----------------------------- 6 files changed, 117 insertions(+), 108 deletions(-) create mode 100644 sympl/_core/combine_properties.py diff --git a/sympl/__init__.py b/sympl/__init__.py index 0550c8f..f47ae50 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -14,13 +14,14 @@ from ._core.constants import ( get_constant, set_constant, set_condensible_name, reset_constants, get_constants_string) -from ._core.tracers import register_tracer, get_tracer_unit_dict, get_tracer_names +from ._core.tracers import ( + register_tracer, get_tracer_unit_dict, get_tracer_input_properties, get_tracer_names) from ._core.util import ( ensure_no_shared_keys, get_numpy_array, jit, restore_dimensions, - get_component_aliases, - combine_component_properties) + get_component_aliases) +from sympl._core.combine_properties import combine_component_properties from ._core.units import units_are_same, units_are_compatible, is_valid_unit from sympl._core.get_np_arrays import get_numpy_arrays_with_properties from sympl._core.restore_dataarray import restore_data_arrays_with_properties @@ -47,7 +48,7 @@ get_constants_string, TimeDifferencingWrapper, ensure_no_shared_keys, get_numpy_array, jit, - register_tracer, get_tracer_unit_dict, get_tracer_names, + register_tracer, get_tracer_unit_dict, get_tracer_input_properties, get_tracer_names, restore_dimensions, get_numpy_arrays_with_properties, restore_data_arrays_with_properties, initialize_numpy_arrays_with_properties, diff --git a/sympl/_core/combine_properties.py b/sympl/_core/combine_properties.py new file mode 100644 index 0000000..fd74402 --- /dev/null +++ b/sympl/_core/combine_properties.py @@ -0,0 +1,101 @@ +from .exceptions import InvalidPropertyDictError +from .tracers import get_tracer_input_properties +from .units import units_are_compatible + + +def combine_dims(dims1, dims2): + """ + Takes in two dims specifications and returns a single specification that + satisfies both, if possible. Raises an InvalidPropertyDictError if not. + + Parameters + ---------- + dims1 : iterable of str + dims2 : iterable of str + + Returns + ------- + dims : iterable of str + + Raises + ------ + InvalidPropertyDictError + If the two dims specifications cannot be combined + """ + if dims1 == dims2: + return dims1 + dims_out = [] + dims1 = set(dims1) + dims2 = set(dims2) + dims1_wildcard = '*' in dims1 + dims1.discard('*') + dims2_wildcard = '*' in dims2 + dims2.discard('*') + unmatched_dims = set(dims1).union(dims2).difference(dims_out) + shared_dims = set(dims2).intersection(dims2) + if dims1_wildcard and dims2_wildcard: + dims_out.insert(0, '*') # either dim can match anything + dims_out.extend(unmatched_dims) + elif not dims1_wildcard and not dims2_wildcard: + if shared_dims != set(dims1) or shared_dims != set(dims2): + raise InvalidPropertyDictError( + 'dims {} and {} are incompatible'.format(dims1, dims2)) + dims_out.extend(unmatched_dims) + elif dims1_wildcard: + if shared_dims != set(dims2): + raise InvalidPropertyDictError( + 'dims {} and {} are incompatible'.format(dims1, dims2)) + dims_out.extend(unmatched_dims) + elif dims2_wildcard: + if shared_dims != set(dims1): + raise InvalidPropertyDictError( + 'dims {} and {} are incompatible'.format(dims1, dims2)) + dims_out.extend(unmatched_dims) + return dims_out + + +def combine_component_properties(component_list, property_name, input_properties=None): + property_list = [] + for component in component_list: + property_list.append(getattr(component, property_name)) + if property_name == 'input_properties' and getattr(component, 'uses_tracers', False): + property_list.append(get_tracer_input_properties(getattr(component, 'prepend_tracers', ()), component.tracer_dims)) + return combine_properties(property_list, input_properties) + + +def combine_properties(property_list, input_properties=None): + if input_properties is None: + input_properties = {} + return_dict = {} + for property_dict in property_list: + for name, properties in property_dict.items(): + if name not in return_dict: + return_dict[name] = {} + return_dict[name].update(properties) + if 'dims' not in properties.keys(): + if name in input_properties.keys() and 'dims' in input_properties[name].keys(): + return_dict[name]['dims'] = input_properties[name]['dims'] + else: + raise InvalidPropertyDictError() + elif not units_are_compatible( + properties['units'], return_dict[name]['units']): + raise InvalidPropertyDictError( + 'Cannot combine components with incompatible units ' + '{} and {} for quantity {}'.format( + return_dict[name]['units'], + properties['units'], name)) + else: + if 'dims' in properties.keys(): + new_dims = properties['dims'] + elif name in input_properties.keys() and 'dims' in input_properties[name].keys(): + new_dims = input_properties[name]['dims'] + else: + raise InvalidPropertyDictError() + try: + dims = combine_dims(return_dict[name]['dims'], new_dims) + return_dict[name]['dims'] = dims + except InvalidPropertyDictError as err: + raise InvalidPropertyDictError( + 'Incompatibility between dims of quantity {}: {}'.format( + name, err.args[0])) + return return_dict diff --git a/sympl/_core/composite.py b/sympl/_core/composite.py index 080a5b3..383a8fb 100644 --- a/sympl/_core/composite.py +++ b/sympl/_core/composite.py @@ -1,7 +1,7 @@ from .base_components import TendencyComponent, DiagnosticComponent, Monitor, ImplicitTendencyComponent from .util import ( - update_dict_by_adding_another, ensure_no_shared_keys, - combine_component_properties) + update_dict_by_adding_another, ensure_no_shared_keys) +from .combine_properties import combine_component_properties from .exceptions import InvalidPropertyDictError diff --git a/sympl/_core/prognosticstepper.py b/sympl/_core/prognosticstepper.py index 1dab6ca..4c48d23 100644 --- a/sympl/_core/prognosticstepper.py +++ b/sympl/_core/prognosticstepper.py @@ -1,7 +1,7 @@ import abc from .composite import ImplicitTendencyComponentComposite from .time import timedelta -from .util import combine_component_properties, combine_properties +from .combine_properties import combine_properties, combine_component_properties from .units import clean_units from .state import copy_untouched_quantities from .base_components import ImplicitTendencyComponent, Stepper diff --git a/sympl/_core/tracers.py b/sympl/_core/tracers.py index ee2150e..1a7a0b6 100644 --- a/sympl/_core/tracers.py +++ b/sympl/_core/tracers.py @@ -67,13 +67,15 @@ def get_tracer_names(): return tuple(_tracer_names) -def get_tracer_properties(prepend_tracers, tracer_dims): +def get_tracer_input_properties(prepend_tracers, tracer_dims): """ Args: prepend_tracers (list of tuple): Pairs of (name, units) describing tracers that are to be included in addition to any registered tracers. + tracer_dims (list): Dimensions to use for each tracer + (e.g. ['dim1', 'dim2']). Returns: input_properties (dict): A properties dictionary for registered and @@ -182,7 +184,7 @@ def pack(self, state): dimensions as specified by tracer_dims on initializing this object. """ - tracer_properties = get_tracer_properties( + tracer_properties = get_tracer_input_properties( self._prepend_tracers, self._tracer_quantity_dims) raw_state = get_numpy_arrays_with_properties(state, tracer_properties) if len(self.tracer_names) == 0: @@ -215,7 +217,7 @@ def unpack(self, tracer_array, input_state, multiply_unit=''): values are DataArrays containing the values of each tracer. """ - tracer_properties = get_tracer_properties( + tracer_properties = get_tracer_input_properties( self._prepend_tracers, self._tracer_quantity_dims) raw_state = {} for i, name in enumerate(self.tracer_names): diff --git a/sympl/_core/util.py b/sympl/_core/util.py index a26fcc8..3745359 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -1,9 +1,10 @@ from datetime import datetime + import numpy as np -from .units import units_are_compatible + from .dataarray import DataArray from .exceptions import ( - SharedKeyError, InvalidPropertyDictError) + SharedKeyError) try: from numba import jit @@ -191,57 +192,6 @@ def same_list(list1, list2): [item in list2 for item in list1] + [item in list1 for item in list2])) -def combine_dims(dims1, dims2): - """ - Takes in two dims specifications and returns a single specification that - satisfies both, if possible. Raises an InvalidPropertyDictError if not. - - Parameters - ---------- - dims1 : iterable of str - dims2 : iterable of str - - Returns - ------- - dims : iterable of str - - Raises - ------ - InvalidPropertyDictError - If the two dims specifications cannot be combined - """ - if dims1 == dims2: - return dims1 - dims_out = [] - dims1 = set(dims1) - dims2 = set(dims2) - dims1_wildcard = '*' in dims1 - dims1.discard('*') - dims2_wildcard = '*' in dims2 - dims2.discard('*') - unmatched_dims = set(dims1).union(dims2).difference(dims_out) - shared_dims = set(dims2).intersection(dims2) - if dims1_wildcard and dims2_wildcard: - dims_out.insert(0, '*') # either dim can match anything - dims_out.extend(unmatched_dims) - elif not dims1_wildcard and not dims2_wildcard: - if shared_dims != set(dims1) or shared_dims != set(dims2): - raise InvalidPropertyDictError( - 'dims {} and {} are incompatible'.format(dims1, dims2)) - dims_out.extend(unmatched_dims) - elif dims1_wildcard: - if shared_dims != set(dims2): - raise InvalidPropertyDictError( - 'dims {} and {} are incompatible'.format(dims1, dims2)) - dims_out.extend(unmatched_dims) - elif dims2_wildcard: - if shared_dims != set(dims1): - raise InvalidPropertyDictError( - 'dims {} and {} are incompatible'.format(dims1, dims2)) - dims_out.extend(unmatched_dims) - return dims_out - - def update_dict_by_adding_another(dict1, dict2): """ Takes two dictionaries. Add values in dict2 to the values in dict1, if @@ -373,51 +323,6 @@ def get_final_shape(data_array, out_dims, direction_to_names): return final_shape -def combine_component_properties(component_list, property_name, input_properties=None): - property_list = [] - for component in component_list: - property_list.append(getattr(component, property_name)) - return combine_properties(property_list, input_properties) - - -def combine_properties(property_list, input_properties=None): - if input_properties is None: - input_properties = {} - return_dict = {} - for property_dict in property_list: - for name, properties in property_dict.items(): - if name not in return_dict: - return_dict[name] = {} - return_dict[name].update(properties) - if 'dims' not in properties.keys(): - if name in input_properties.keys() and 'dims' in input_properties[name].keys(): - return_dict[name]['dims'] = input_properties[name]['dims'] - else: - raise InvalidPropertyDictError() - elif not units_are_compatible( - properties['units'], return_dict[name]['units']): - raise InvalidPropertyDictError( - 'Cannot combine components with incompatible units ' - '{} and {} for quantity {}'.format( - return_dict[name]['units'], - properties['units'], name)) - else: - if 'dims' in properties.keys(): - new_dims = properties['dims'] - elif name in input_properties.keys() and 'dims' in input_properties[name].keys(): - new_dims = input_properties[name]['dims'] - else: - raise InvalidPropertyDictError() - try: - dims = combine_dims(return_dict[name]['dims'], new_dims) - return_dict[name]['dims'] = dims - except InvalidPropertyDictError as err: - raise InvalidPropertyDictError( - 'Incompatibility between dims of quantity {}: {}'.format( - name, err.args[0])) - return return_dict - - def get_component_aliases(*args): """ Returns aliases for variables in the properties of Components (TendencyComponent, From 52b23dd6d840df9c7b160b931a121cd793ee220b Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 30 Jul 2018 21:44:15 -0700 Subject: [PATCH 87/98] Moved property code to avoid circular dependency --- tests/test_util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_util.py b/tests/test_util.py index 5951161..d24330c 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -5,7 +5,9 @@ TendencyComponent, ensure_no_shared_keys, SharedKeyError, DataArray, Stepper, DiagnosticComponent, InvalidPropertyDictError) -from sympl._core.util import update_dict_by_adding_another, combine_dims, get_component_aliases +from sympl._core.util import update_dict_by_adding_another, \ + get_component_aliases +from sympl._core.combine_properties import combine_dims def same_list(list1, list2): From a240c7fffb96ff305b279fb960a4563830f66b96 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 6 Aug 2018 16:10:57 -0700 Subject: [PATCH 88/98] Renamed PrognosticStepper to TendencyStepper --- HISTORY.rst | 6 ++-- docs/computation.rst | 12 ++++---- docs/overview.rst | 6 ++-- docs/quickstart.rst | 8 +++--- docs/state.rst | 2 +- docs/timestepping.rst | 18 ++++++------ sympl/__init__.py | 4 +-- sympl/_components/timesteppers.py | 14 +++++----- ...rognosticstepper.py => tendencystepper.py} | 28 +++++++++---------- 9 files changed, 49 insertions(+), 49 deletions(-) rename sympl/_core/{prognosticstepper.py => tendencystepper.py} (91%) diff --git a/HISTORY.rst b/HISTORY.rst index e2dc0ef..ea6e6e3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,7 +8,7 @@ Latest * Stepper, DiagnosticComponent, ImplicitTendencyComponent, and TendencyComponent base classes were modified to include functionality that was previously in ScalingWrapper, UpdateFrequencyWrapper, and TendencyInDiagnosticsWrapper. The functionality of - TendencyInDiagnosticsWrapper is now to be used in Stepper and PrognosticStepper objects. + TendencyInDiagnosticsWrapper is now to be used in Stepper and TendencyStepper objects. * Composites now have a component_list attribute which contains the components being composited. * TimeSteppers now have a prognostic_list attribute which contains the @@ -30,7 +30,7 @@ Latest respectively if outputs do not match. * Added a priority order of property types for determining which aliases are returned by get_component_aliases. -* Fixed a bug where PrognosticStepper objects would modify the arrays passed to them by +* Fixed a bug where TendencyStepper objects would modify the arrays passed to them by TendencyComponent objects, leading to unexpected value changes. * Fixed a bug where constants were missing from the string returned by get_constants_string, particularly any new constants (issue #27) @@ -55,7 +55,7 @@ Breaking changes ~~~~~~~~~~~~~~~~ * Implicit, Timestepper, Prognostic, ImplicitPrognostic, and Diagnostic objects have been renamed to - PrognosticStepper, Stepper, TendencyComponent, ImplicitTendencyComponent, + TendencyStepper, Stepper, TendencyComponent, ImplicitTendencyComponent, and DiagnosticComponent. These changes are also reflected in subclass names. * inputs, outputs, diagnostics, and tendencies are no longer attributes of components. In order to get these, you should use e.g. input_properties.keys() diff --git a/docs/computation.rst b/docs/computation.rst index f28f036..6cad3a9 100644 --- a/docs/computation.rst +++ b/docs/computation.rst @@ -10,7 +10,7 @@ diagnostics at the current time. :py:class:`~sympl.DiagnosticComponent` objects return only diagnostics from the current time. :py:class:`~sympl.Stepper` objects will take in a timestep along with the state, and then return the next state as well as modifying the current state to include more diagnostics -(it is similar to a :py:class:`~sympl.PrognosticStepper` in how it is called). +(it is similar to a :py:class:`~sympl.TendencyStepper` in how it is called). In specific cases, it may be necessary to use a :py:class:`~sympl.ImplicitTendencyComponent` object, which is discussed at the end of this section. @@ -32,7 +32,7 @@ TendencyComponent As stated above, :py:class:`~sympl.TendencyComponent` objects use the state to return tendencies and diagnostics at the current time. In a full model, the tendencies -are used by a time stepping scheme (in Sympl, a :py:class:`~sympl.PrognosticStepper`) +are used by a time stepping scheme (in Sympl, a :py:class:`~sympl.TendencyStepper`) to determine the values of quantities at the next time. You can call a :py:class:`~sympl.TendencyComponent` directly to get diagnostics and @@ -49,7 +49,7 @@ does not compute any diagnostics, it will still return an empty diagnostics dictionary. Usually, you will call a TendencyComponent object through a -:py:class:`~sympl.PrognosticStepper` that uses it to determine values at the next +:py:class:`~sympl.TendencyStepper` that uses it to determine values at the next timestep. .. autoclass:: sympl.TendencyComponent @@ -301,14 +301,14 @@ The reason to avoid using an :py:class:`~sympl.ImplicitTendencyComponent` is tha a component requires a timestep, it is making internal assumptions about how you are timestepping. For example, it may use the timestep to ensure that all supersaturated water is condensed by the end of the timestep using an assumption -about the timestepping. However, if you use a :py:class:`~sympl.PrognosticStepper` +about the timestepping. However, if you use a :py:class:`~sympl.TendencyStepper` which does not obey those assumptions, you may get unintended behavior, such as some supersaturated water remaining, or too much water being condensed. -For this reason, the :py:class:`~sympl.PrognosticStepper` objects included in Sympl +For this reason, the :py:class:`~sympl.TendencyStepper` objects included in Sympl do not wrap :py:class:`~sympl.ImplicitTendencyComponent` components. If you would like to use this type of component, and know what you are doing, it is pretty easy -to write your own :py:class:`~sympl.PrognosticStepper` to do so (you can base the code +to write your own :py:class:`~sympl.TendencyStepper` to do so (you can base the code off of the code in Sympl), or the model you are using might already have components to do this for you. diff --git a/docs/overview.rst b/docs/overview.rst index 7a35996..567c81f 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -73,17 +73,17 @@ set to initial values. Code to do this may be present in other packages, or you can write this code yourself. The state and its initialization is discussed further in :ref:`Model State`. -The state dictionary is evolved by :py:class:`~sympl.PrognosticStepper` and +The state dictionary is evolved by :py:class:`~sympl.TendencyStepper` and :py:class:`~sympl.Stepper` objects. These types of objects take in the state and a timedelta object that indicates the time step, and return the next -model state. :py:class:`~sympl.PrognosticStepper` objects do this by wrapping +model state. :py:class:`~sympl.TendencyStepper` objects do this by wrapping :py:class:`~sympl.TendencyComponent` objects, which calculate tendencies using the state dictionary. We should note that the meaning of "Stepper" in Sympl is slightly different than its traditional definition. Here an "Stepper" object is one that calculates the new state directly from the current state, or any object that requires the timestep to calculate the new state, while "TendencyComponent" objects are ones that calculate tendencies without using the -timestep. If a :py:class:`~sympl.PrognosticStepper` or :py:class:`~sympl.Stepper` +timestep. If a :py:class:`~sympl.TendencyStepper` or :py:class:`~sympl.Stepper` object needs to use multiple time steps in its calculation, it does so by storing states it was previously given until they are no longer needed. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 8d19d19..7e6bb48 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -137,13 +137,13 @@ Those are the "components": ]) implicit_dynamics = ImplicitDynamics() -:py:class:`~sympl.AdamsBashforth` is a :py:class:`~sympl.PrognosticStepper`, which is +:py:class:`~sympl.AdamsBashforth` is a :py:class:`~sympl.TendencyStepper`, which is created with a set of :py:class:`~sympl.TendencyComponent` components. The :py:class:`~sympl.TendencyComponent` components we have here are ``Radiation``, ``BoundaryLayer``, and ``DeepConvection``. Each of these carries information about what it takes as inputs and provides as outputs, and can be called with a model state to return tendencies for a set of quantities. The -:py:class:`~sympl.PrognosticStepper` uses this information to step the model state +:py:class:`~sympl.TendencyStepper` uses this information to step the model state forward in time. The :py:class:`~sympl.UpdateFrequencyWrapper` applied to the ``Radiation`` object @@ -155,7 +155,7 @@ calculations (which are very expensive) every timestep, but it can be applied to any TendencyComponent. The :py:class:`ImplicitDynamics` class is a :py:class:`~sympl.Stepper` object, which -steps the model state forward in time in the same way that a :py:class:`~sympl.PrognosticStepper` +steps the model state forward in time in the same way that a :py:class:`~sympl.TendencyStepper` would, but doesn't use :py:class:`~sympl.TendencyComponent` objects in doing so. The Main Loop @@ -180,6 +180,6 @@ In the main loop, a series of component calls update the state, and the figure presented by ``plot_monitor`` is updated. The code is meant to be as self-explanatory as possible. It is necessary to manually set the time of the next state at the end of the loop. This is not done automatically by -:py:class:`~sympl.PrognosticStepper` and :py:class:`~sympl.Stepper` objects, because +:py:class:`~sympl.TendencyStepper` and :py:class:`~sympl.Stepper` objects, because in many models you may want to update the state with multiple such objects in a sequence over the course of a single time step. diff --git a/docs/state.rst b/docs/state.rst index e61f0b3..26e00ab 100644 --- a/docs/state.rst +++ b/docs/state.rst @@ -11,7 +11,7 @@ subtraction, and contains a helpful method for converting units. Any information about the grid the data is using that components need should be put as attributes in the ``attrs`` of the ``DataArray`` objects. Deciding on these attributes (if any) is mostly up to the component developers. However, -in order to use the PrognosticStepper objects and several helper functions from Sympl, +in order to use the TendencyStepper objects and several helper functions from Sympl, it is required that a "units" attribute is present. .. _xarray: http://xarray.pydata.org/en/stable/ diff --git a/docs/timestepping.rst b/docs/timestepping.rst index 70f140f..517f826 100644 --- a/docs/timestepping.rst +++ b/docs/timestepping.rst @@ -1,7 +1,7 @@ Timestepping ============ -:py:class:`~sympl.PrognosticStepper` objects use time derivatives from +:py:class:`~sympl.TendencyStepper` objects use time derivatives from :py:class:`~sympl.TendencyComponent` objects to step a model state forward in time. They are initialized using any number of :py:class:`~sympl.TendencyComponent` objects. @@ -10,7 +10,7 @@ They are initialized using any number of :py:class:`~sympl.TendencyComponent` ob from sympl import AdamsBashforth time_stepper = AdamsBashforth(MyPrognostic(), MyOtherPrognostic()) -Once initialized, a :py:class:`~sympl.PrognosticStepper` object has a very similar +Once initialized, a :py:class:`~sympl.TendencyStepper` object has a very similar interface to the :py:class:`~sympl.Stepper` object. .. code-block:: python @@ -29,32 +29,32 @@ and that they have been modified. In other words, ``state`` may be modified by this call. For instance, the time filtering necessary when using Leapfrog time stepping means the current model state has to be modified by the filter. -It is only after calling the :py:class:`~sympl.PrognosticStepper` and getting the +It is only after calling the :py:class:`~sympl.TendencyStepper` and getting the diagnostics that you will have a complete state with all diagnostic quantities. This means you will sometimes want to pass ``state`` to your :py:class:`~sympl.Monitor` objects *after* calling -the :py:class:`~sympl.PrognosticStepper` and getting ``next_state``. +the :py:class:`~sympl.TendencyStepper` and getting ``next_state``. -.. warning:: :py:class:`~sympl.PrognosticStepper` objects do not, and should not, +.. warning:: :py:class:`~sympl.TendencyStepper` objects do not, and should not, update 'time' in the model state. -Keep in mind that for split-time models, multiple :py:class:`~sympl.PrognosticStepper` +Keep in mind that for split-time models, multiple :py:class:`~sympl.TendencyStepper` objects might be called in in a single pass of the main loop. If each one updated ``state['time']``, the time would be moved forward more than it should. -For that reason, :py:class:`~sympl.PrognosticStepper` objects do not update +For that reason, :py:class:`~sympl.TendencyStepper` objects do not update ``state['time']``. There are also :py:class:`~sympl.Stepper` objects which evolve the state forward in time without the use of TendencyComponent objects. These function exactly the same as a -:py:class:`~sympl.PrognosticStepper` once they are created, but do not accept +:py:class:`~sympl.TendencyStepper` once they are created, but do not accept :py:class:`~sympl.TendencyComponent` objects when you create them. One example might be a component that condenses all supersaturated moisture over some time period. :py:class:`~sympl.Stepper` objects are generally used for parameterizations that work by determining the target model state in some way, or involve limiters, and cannot be represented as a :py:class:`~sympl.TendencyComponent`. -.. autoclass:: sympl.PrognosticStepper +.. autoclass:: sympl.TendencyStepper :members: :special-members: :exclude-members: __weakref__,__metaclass__ diff --git a/sympl/__init__.py b/sympl/__init__.py index f47ae50..25ca3d2 100644 --- a/sympl/__init__.py +++ b/sympl/__init__.py @@ -4,7 +4,7 @@ ) from ._core.composite import TendencyComponentComposite, DiagnosticComponentComposite, \ MonitorComposite, ImplicitTendencyComponentComposite -from ._core.prognosticstepper import PrognosticStepper +from ._core.tendencystepper import TendencyStepper from ._components.timesteppers import AdamsBashforth, Leapfrog, SSPRungeKutta from ._core.exceptions import ( InvalidStateError, SharedKeyError, DependencyError, @@ -38,7 +38,7 @@ TendencyComponent, DiagnosticComponent, Stepper, Monitor, TendencyComponentComposite, ImplicitTendencyComponentComposite, DiagnosticComponentComposite, MonitorComposite, ImplicitTendencyComponent, - PrognosticStepper, Leapfrog, AdamsBashforth, SSPRungeKutta, + TendencyStepper, Leapfrog, AdamsBashforth, SSPRungeKutta, InvalidStateError, SharedKeyError, DependencyError, InvalidPropertyDictError, ComponentExtraOutputError, ComponentMissingOutputError, diff --git a/sympl/_components/timesteppers.py b/sympl/_components/timesteppers.py index 5c84f6a..e76f53e 100644 --- a/sympl/_components/timesteppers.py +++ b/sympl/_components/timesteppers.py @@ -1,11 +1,11 @@ -from .._core.prognosticstepper import PrognosticStepper +from .._core.tendencystepper import TendencyStepper from .._core.dataarray import DataArray from .._core.state import copy_untouched_quantities, add, multiply -class SSPRungeKutta(PrognosticStepper): +class SSPRungeKutta(TendencyStepper): """ - A PrognosticStepper using the Strong Stability Preserving Runge-Kutta scheme, + A TendencyStepper using the Strong Stability Preserving Runge-Kutta scheme, as in Numerical Methods for Fluid Dynamics by Dale Durran (2nd ed) and as proposed by Shu and Osher (1988). """ @@ -71,8 +71,8 @@ def _step_2_stages(self, state, timestep): return diagnostics, out_state -class AdamsBashforth(PrognosticStepper): - """A PrognosticStepper using the Adams-Bashforth scheme.""" +class AdamsBashforth(TendencyStepper): + """A TendencyStepper using the Adams-Bashforth scheme.""" def __init__(self, *args, **kwargs): """ @@ -173,8 +173,8 @@ def convert_tendencies_units_for_state(tendencies, state): tendencies[quantity_name] = tendencies[quantity_name].to_units(desired_units) -class Leapfrog(PrognosticStepper): - """A PrognosticStepper using the Leapfrog scheme. +class Leapfrog(TendencyStepper): + """A TendencyStepper using the Leapfrog scheme. This scheme calculates the values at time $t_{n+1}$ using the derivatives at $t_{n}$ and values at diff --git a/sympl/_core/prognosticstepper.py b/sympl/_core/tendencystepper.py similarity index 91% rename from sympl/_core/prognosticstepper.py rename to sympl/_core/tendencystepper.py index 4c48d23..703af00 100644 --- a/sympl/_core/prognosticstepper.py +++ b/sympl/_core/tendencystepper.py @@ -9,7 +9,7 @@ import warnings -class PrognosticStepper(Stepper): +class TendencyStepper(Stepper): """An object which integrates model state forward in time. It uses TendencyComponent and DiagnosticComponent objects to update the current model state @@ -29,11 +29,11 @@ class PrognosticStepper(Stepper): 'units'. prognostic : ImplicitTendencyComponentComposite A composite of the TendencyComponent and ImplicitPrognostic objects used by - the PrognosticStepper. + the TendencyStepper. prognostic_list: list of TendencyComponent and ImplicitPrognosticComponent - A list of TendencyComponent objects called by the PrognosticStepper. These should + A list of TendencyComponent objects called by the TendencyStepper. These should be referenced when determining what inputs are necessary for the - PrognosticStepper. + TendencyStepper. tendencies_in_diagnostics : bool A boolean indicating whether this object will put tendencies of quantities in its diagnostic output. @@ -94,7 +94,7 @@ def _tendency_properties(self): def __str__(self): return ( - 'instance of {}(PrognosticStepper)\n' + 'instance of {}(TendencyStepper)\n' ' TendencyComponent components: {}'.format(self.prognostic_list) ) @@ -112,11 +112,11 @@ def __repr__(self): return return_value def array_call(self, state, timestep): - raise NotImplementedError('PrognosticStepper objects do not implement array_call') + raise NotImplementedError('TendencyStepper objects do not implement array_call') def __init__(self, *args, **kwargs): """ - Initialize the PrognosticStepper. + Initialize the TendencyStepper. Parameters ---------- @@ -138,16 +138,16 @@ def __init__(self, *args, **kwargs): args = args[0] if any(isinstance(a, ImplicitTendencyComponent) for a in args): warnings.warn( - 'Using an ImplicitTendencyComponent in sympl PrognosticStepper objects may ' + 'Using an ImplicitTendencyComponent in sympl TendencyStepper objects may ' 'lead to scientifically invalid results. Make sure the component ' - 'follows the same numerical assumptions as the PrognosticStepper used.') + 'follows the same numerical assumptions as the TendencyStepper used.') self.prognostic = ImplicitTendencyComponentComposite(*args) - super(PrognosticStepper, self).__init__(**kwargs) + super(TendencyStepper, self).__init__(**kwargs) for name in self.prognostic.tendency_properties.keys(): if name not in self.output_properties.keys(): raise InvalidPropertyDictError( 'Prognostic object has tendency output for {} but ' - 'PrognosticStepper containing it does not have it in ' + 'TendencyStepper containing it does not have it in ' 'output_properties.'.format(name)) self.__initialized = True @@ -183,7 +183,7 @@ def __call__(self, state, timestep): """ if not self.__initialized: raise AssertionError( - 'PrognosticStepper component has not had its base class ' + 'TendencyStepper component has not had its base class ' '__init__ called, likely due to a missing call to ' 'super(ClassName, self).__init__(*args, **kwargs) in its ' '__init__ method.' @@ -205,8 +205,8 @@ def _insert_tendencies_to_diagnostics( raise RuntimeError( 'A TendencyComponent has output tendencies as a diagnostic and has' ' caused a name clash when trying to do so from this ' - 'PrognosticStepper ({}). You must disable ' - 'tendencies_in_diagnostics for this PrognosticStepper.'.format( + 'TendencyStepper ({}). You must disable ' + 'tendencies_in_diagnostics for this TendencyStepper.'.format( tendency_name)) base_units = input_properties[name]['units'] diagnostics[tendency_name] = ( From cf59604af34ef2daf9518ccdcdb9bdaa53307e40 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 6 Aug 2018 17:01:13 -0700 Subject: [PATCH 89/98] Fixed bug in update_dict_by_adding_another Bug would take place when initial dictionary had a DataArray which does not have a dimension in the second dictionary. In that case in-place addition cannot take place because the dimensions of the first array are not mutable. Fixed by replacing the array with a new array (the dict is still updated in-place with the new array). --- sympl/_core/util.py | 13 +++++++--- tests/test_util.py | 60 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/sympl/_core/util.py b/sympl/_core/util.py index 3745359..3cc687b 100644 --- a/sympl/_core/util.py +++ b/sympl/_core/util.py @@ -4,7 +4,7 @@ from .dataarray import DataArray from .exceptions import ( - SharedKeyError) + SharedKeyError, InvalidStateError) try: from numba import jit @@ -207,9 +207,14 @@ def update_dict_by_adding_another(dict1, dict2): else: dict1[key] = dict2[key] else: - if (isinstance(dict1[key], DataArray) and isinstance(dict2[key], DataArray) and - ('units' in dict1[key].attrs) and ('units' in dict2[key].attrs)): - dict1[key] += dict2[key].to_units(dict1[key].attrs['units']) + if (isinstance(dict1[key], DataArray) and isinstance(dict2[key], DataArray)): + if 'units' not in dict1[key].attrs or 'units' not in dict2[key].attrs: + raise InvalidStateError( + 'DataArray objects must have units property defined') + try: + dict1[key] += dict2[key].to_units(dict1[key].attrs['units']) + except ValueError: # dict1[key] is missing a dimension present in dict2[key] + dict1[key] = dict1[key] + dict2[key].to_units(dict1[key].attrs['units']) else: dict1[key] += dict2[key] # += is in-place addition operator return # not returning anything emphasizes that this is in-place diff --git a/tests/test_util.py b/tests/test_util.py index d24330c..663b4c8 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -38,6 +38,66 @@ def __init__(self, input_properties, diagnostic_properties): self.diagnostic_properties = diagnostic_properties +def test_update_dict_by_adding_another_works_on_different_dim_orders(): + dict1 = { + 'quantity': DataArray( + np.ones([2, 3, 4]), + dims=['dim1', 'dim2', 'dim3'], + attrs={'units': 'm'}, + ) + } + dict2 = { + 'quantity': DataArray( + np.ones([3, 2, 4]), + dims=['dim2', 'dim1', 'dim3'], + attrs={'units': 'm'}, + ) + } + update_dict_by_adding_another(dict1, dict2) + assert dict1['quantity'].shape == (2, 3, 4) + assert np.all(dict1['quantity'].values == 2.) + + +def test_update_dict_by_adding_another_broadcasts_added_dim(): + dict1 = { + 'quantity': DataArray( + np.ones([2, 3, 4]), + dims=['dim1', 'dim2', 'dim3'], + attrs={'units': 'm'}, + ) + } + dict2 = { + 'quantity': DataArray( + np.ones([3, 2]), + dims=['dim2', 'dim1'], + attrs={'units': 'm'}, + ) + } + update_dict_by_adding_another(dict1, dict2) + assert dict1['quantity'].shape == (2, 3, 4) + assert np.all(dict1['quantity'].values == 2.) + + +def test_update_dict_by_adding_another_broadcasts_initial_dim(): + dict1 = { + 'quantity': DataArray( + np.ones([2, 3]), + dims=['dim1', 'dim2'], + attrs={'units': 'm'}, + ) + } + dict2 = { + 'quantity': DataArray( + np.ones([3, 2, 4]), + dims=['dim2', 'dim1', 'dim3'], + attrs={'units': 'm'}, + ) + } + update_dict_by_adding_another(dict1, dict2) + assert dict1['quantity'].shape == (2, 3, 4) + assert np.all(dict1['quantity'].values == 2.) + + def test_update_dict_by_adding_another_adds_shared_arrays(): old_a = np.array([1., 1.]) dict1 = {'a': old_a} From 3277b7bd776a5841e9667c178c357548fdd5f344 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 6 Aug 2018 17:03:52 -0700 Subject: [PATCH 90/98] Fixed bugs in properties creation for TimeDifferencingWrapper --- sympl/_components/basic.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sympl/_components/basic.py b/sympl/_components/basic.py index f06648f..7897d67 100644 --- a/sympl/_components/basic.py +++ b/sympl/_components/basic.py @@ -378,13 +378,15 @@ def input_properties(self): @property def tendency_properties(self): - return_dict = self._implicit.output_properties.copy() - return_dict.update(self._tendency_diagnostic_properties) + return_dict = {} + for name, properties in self._implicit.output_properties.items(): + return_dict[name] = properties.copy() + return_dict[name]['units'] += ' s^-1' return return_dict @property def diagnostic_properties(self): - return self._implicit.diagnostic_propertes + return self._implicit.diagnostic_properties def __init__(self, implicit, **kwargs): """ From 3eeaee0f31a0cee42db38106e017bd8458d4bf3b Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Sun, 12 Aug 2018 12:45:10 -0700 Subject: [PATCH 91/98] Fixed bug in TendencyStepper property generation --- sympl/_core/tendencystepper.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/sympl/_core/tendencystepper.py b/sympl/_core/tendencystepper.py index 703af00..65950c7 100644 --- a/sympl/_core/tendencystepper.py +++ b/sympl/_core/tendencystepper.py @@ -58,16 +58,19 @@ def input_properties(self): self.prognostic_list, 'input_properties') return combine_properties([input_properties, self.output_properties]) + @property + def _tendencycomponent_input_properties(self): + return combine_component_properties( + self.prognostic_list, 'input_properties') + @property def diagnostic_properties(self): return_value = {} for prognostic in self.prognostic_list: return_value.update(prognostic.diagnostic_properties) if self.tendencies_in_diagnostics: - tendency_properties = combine_component_properties( - self.prognostic_list, 'tendency_properties') self._insert_tendencies_to_diagnostic_properties( - return_value, tendency_properties) + return_value, self._tendency_properties) return return_value def _insert_tendencies_to_diagnostic_properties( @@ -81,8 +84,7 @@ def _insert_tendencies_to_diagnostic_properties( @property def output_properties(self): - output_properties = combine_component_properties( - self.prognostic_list, 'tendency_properties') + output_properties = self._tendency_properties for name, properties in output_properties.items(): properties['units'] += ' {}'.format(self.time_unit_name) properties['units'] = clean_units(properties['units']) @@ -90,7 +92,12 @@ def output_properties(self): @property def _tendency_properties(self): - return combine_component_properties(self.prognostic_list, 'tendency_properties') + return_dict = {} + return_dict.update(combine_component_properties( + self.prognostic_list, 'tendency_properties', + input_properties=self._tendencycomponent_input_properties + )) + return return_dict def __str__(self): return ( From 59a5a873e2b97d3afcaf2be086a177d396e9db63 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Mon, 13 Aug 2018 11:59:49 -0700 Subject: [PATCH 92/98] Fixed property combination so it does not add tracer dimension to tracer quantity properties --- sympl/_core/combine_properties.py | 9 ++++++++- sympl/_core/tracers.py | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/sympl/_core/combine_properties.py b/sympl/_core/combine_properties.py index fd74402..623a3cf 100644 --- a/sympl/_core/combine_properties.py +++ b/sympl/_core/combine_properties.py @@ -59,7 +59,14 @@ def combine_component_properties(component_list, property_name, input_properties for component in component_list: property_list.append(getattr(component, property_name)) if property_name == 'input_properties' and getattr(component, 'uses_tracers', False): - property_list.append(get_tracer_input_properties(getattr(component, 'prepend_tracers', ()), component.tracer_dims)) + tracer_dims = list(component.tracer_dims) + if 'tracer' not in tracer_dims: + raise InvalidPropertyDictError( + "tracer_dims must include a 'tracer' dimension indicating " + "tracer number" + ) + tracer_dims.remove('tracer') + property_list.append(get_tracer_input_properties(getattr(component, 'prepend_tracers', ()), tracer_dims)) return combine_properties(property_list, input_properties) diff --git a/sympl/_core/tracers.py b/sympl/_core/tracers.py index 1a7a0b6..b84c853 100644 --- a/sympl/_core/tracers.py +++ b/sympl/_core/tracers.py @@ -81,6 +81,9 @@ def get_tracer_input_properties(prepend_tracers, tracer_dims): input_properties (dict): A properties dictionary for registered and additional tracers. """ + tracer_dims = list(tracer_dims) + if 'tracer' in tracer_dims: + tracer_dims.remove('tracer') tracer_units = {} tracer_units.update(get_tracer_unit_dict()) tracer_units.update(dict(prepend_tracers)) From de63029dfcac4312f6fd24d51b1da991063e191f Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 15 Aug 2018 10:55:54 -0700 Subject: [PATCH 93/98] Fixed init args of dummy component --- sympl/_components/netcdf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sympl/_components/netcdf.py b/sympl/_components/netcdf.py index 29d7984..6a856b1 100644 --- a/sympl/_components/netcdf.py +++ b/sympl/_components/netcdf.py @@ -9,6 +9,7 @@ import numpy as np from datetime import timedelta from six import string_types + try: import netCDF4 as nc4 except ImportError: @@ -19,7 +20,7 @@ # user they need to install the dependency if they try to use it class NetCDFMonitor(Monitor): - def __init__(self, filename): + def __init__(self, *args, **kwargs): raise DependencyError( 'netCDF4-python must be installed to use NetCDFMonitor') From 4f323f7fb309e76ce0ef9ad919ad862b8742f7b9 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 15 Aug 2018 11:00:25 -0700 Subject: [PATCH 94/98] Moved nc4 missing error to real class and removed dummy class --- sympl/_components/netcdf.py | 373 ++++++++++++++++++------------------ 1 file changed, 183 insertions(+), 190 deletions(-) diff --git a/sympl/_components/netcdf.py b/sympl/_components/netcdf.py index 6a856b1..23ef016 100644 --- a/sympl/_components/netcdf.py +++ b/sympl/_components/netcdf.py @@ -15,200 +15,190 @@ except ImportError: nc4 = None -if nc4 is None: - # If dependency is not installed, use a dummy object that will alert the - # user they need to install the dependency if they try to use it - class NetCDFMonitor(Monitor): +class NetCDFMonitor(Monitor): + """A Monitor which caches stored states and then writes them to a + NetCDF file when requested.""" - def __init__(self, *args, **kwargs): + def __init__( + self, filename, time_units='seconds', store_names=None, + write_on_store=False, aliases=None): + """ + Args + ---- + filename : str + The file to which the NetCDF file will be written. + time_units : str, optional + The units in which time will be + stored in the NetCDF file. Time is stored as an integer + number of these units. Default is seconds. + store_names : iterable of str, optional + Names of quantities to store. If not given, + all quantities are stored. + write_on_store : bool, optional + If True, stored changes are immediately written to file. + This can result in many file open/close operations. + Default is to write only when the write() method is + called directly. + aliases : dict + A dictionary of string replacements to apply to state variable + names before saving them in netCDF files. + """ + if nc4 is None: raise DependencyError( 'netCDF4-python must be installed to use NetCDFMonitor') + self._cached_state_dict = {} + self._filename = filename + self._time_units = time_units + self._write_on_store = write_on_store + if aliases is None: + self._aliases = {} + else: + self._aliases = aliases + for key, val in self._aliases.items(): + if not isinstance(key, string_types): + raise TypeError("Bad alias key type: {}. Expected string.".format(type(key))) + elif not isinstance(val, string_types): + raise TypeError("Bad alias value type: {}. Expected string.".format(type(val))) + if store_names is None: + self._store_names = None + else: + self._store_names = ['time'] + list(store_names) - def store(self, state): - pass - -else: - class NetCDFMonitor(Monitor): - """A Monitor which caches stored states and then writes them to a - NetCDF file when requested.""" - - def __init__( - self, filename, time_units='seconds', store_names=None, - write_on_store=False, aliases=None): - """ - Args - ---- - filename : str - The file to which the NetCDF file will be written. - time_units : str, optional - The units in which time will be - stored in the NetCDF file. Time is stored as an integer - number of these units. Default is seconds. - store_names : iterable of str, optional - Names of quantities to store. If not given, - all quantities are stored. - write_on_store : bool, optional - If True, stored changes are immediately written to file. - This can result in many file open/close operations. - Default is to write only when the write() method is - called directly. - aliases : dict - A dictionary of string replacements to apply to state variable - names before saving them in netCDF files. - """ - self._cached_state_dict = {} - self._filename = filename - self._time_units = time_units - self._write_on_store = write_on_store - if aliases is None: - self._aliases = {} - else: - self._aliases = aliases - for key, val in self._aliases.items(): - if not isinstance(key, string_types): - raise TypeError("Bad alias key type: {}. Expected string.".format(type(key))) - elif not isinstance(val, string_types): - raise TypeError("Bad alias value type: {}. Expected string.".format(type(val))) - if store_names is None: - self._store_names = None - else: - self._store_names = ['time'] + list(store_names) - - def store(self, state): - """ - Caches the given state. If write_on_store=True was passed on - initialization, also writes to file. Normally a call to the - write() method is required to write to file. - - Args - ---- - state : dict - A model state dictionary. - - Raises - ------ - InvalidStateError - If state is not a valid input for the DiagnosticComponent instance. - """ - if self._store_names is not None: - name_list = set(state.keys()).intersection(self._store_names) - cache_state = {name: state[name] for name in name_list} - else: - cache_state = state.copy() - - # raise an exception if the state has any empty string variables - for full_var_name in cache_state.keys(): - if len(full_var_name) == 0: - raise ValueError('The given state has an empty string as a variable name.') - - # replace cached variable names with their aliases - for longname, shortname in self._aliases.items(): - for full_var_name in tuple(cache_state.keys()): - # replace any string in the full variable name that matches longname - # example: if longname is "temperature", shortname is "T", and - # full_var_name is "temperature_tendency_from_radiation", the - # alias_name for the variable would be: "T_tendency_from_radiation" - if longname in full_var_name: - alias_name = full_var_name.replace(longname, shortname) - if len(alias_name) == 0: # raise exception if the alias is an empty str - errstr = 'Tried to alias variable "{}" to an empty string.\n' + \ - 'xarray will not allow empty strings as variable names.' - raise ValueError(errstr.format(full_var_name)) - cache_state[alias_name] = cache_state.pop(full_var_name) - - cache_state.pop('time') # stored as key, not needed in state dict - if state['time'] in self._cached_state_dict.keys(): - self._cached_state_dict[state['time']].update(cache_state) - else: - self._cached_state_dict[state['time']] = cache_state - if self._write_on_store: - self.write() - - @property - def _write_mode(self): - if not os.path.isfile(self._filename): - return 'w' - else: - return 'a' - - def _ensure_cached_state_keys_compatible_with_dataset(self, dataset): - file_keys = list(dataset.variables.keys()) - if 'time' in file_keys: - file_keys.remove('time') - if len(file_keys) > 0: - self._ensure_cached_states_have_same_keys(file_keys) - else: - self._ensure_cached_states_have_same_keys() - - def _ensure_cached_states_have_same_keys(self, desired_keys=None): - """ - Ensures all states in self._cached_state_dict have the same keys. - If desired_keys is given, also ensure the keys are the same as - the ones in desired_keys. - - Raises - ------ - InvalidStateError - If the cached states do not meet the requirements. - """ - if len(self._cached_state_dict) == 0: - return # trivially true - if desired_keys is not None: - reference_keys = desired_keys - else: - reference_state = tuple(self._cached_state_dict.values())[0] - reference_keys = reference_state.keys() - for state in self._cached_state_dict.values(): - if not same_list(list(state.keys()), list(reference_keys)): - raise InvalidStateError( - 'NetCDFMonitor was passed a different set of ' - 'quantities for different times: {} vs. {}'.format( - list(reference_keys), list(state.keys()))) - - def _get_ordered_times_and_states(self): - """Returns the items in self._cached_state_dict, sorted by time.""" - return zip(*sorted(self._cached_state_dict.items(), key=lambda x: x[0])) - - def write(self): - """ - Write all cached states to the NetCDF file, and clear the cache. - This will append to any existing NetCDF file. - - Raises - ------ - InvalidStateError - If cached states do not all have the same quantities - as every other cached and written state. - """ - with nc4.Dataset(self._filename, self._write_mode) as dataset: - self._ensure_cached_state_keys_compatible_with_dataset(dataset) - time_list, state_list = self._get_ordered_times_and_states() - self._ensure_time_exists(dataset, time_list[0]) - it_start = dataset.dimensions['time'].size - it_end = it_start + len(time_list) - append_times_to_dataset(time_list, dataset, self._time_units) - all_states = combine_states(state_list) - for name, value in all_states.items(): - ensure_variable_exists(dataset, name, value) - dataset.variables[name][ - it_start:it_end, :] = value.values[:, :] - self._cached_state_dict = {} - - def _ensure_time_exists(self, dataset, possible_reference_time): - """Ensure an unlimited time dimension relevant to this monitor - exists in the NetCDF4 dataset, and create it if it does not.""" - ensure_dimension_exists(dataset, 'time', None) - if 'time' not in dataset.variables: - dataset.createVariable('time', np.int64, ('time',)) - if isinstance(possible_reference_time, timedelta): - dataset.variables['time'].setncattr( - 'units', self._time_units) - else: # assume datetime - dataset.variables['time'].setncattr( - 'units', '{} since {}'.format( - self._time_units, possible_reference_time)) - dataset.variables['time'].setncattr( - 'calendar', 'proleptic_gregorian') + def store(self, state): + """ + Caches the given state. If write_on_store=True was passed on + initialization, also writes to file. Normally a call to the + write() method is required to write to file. + + Args + ---- + state : dict + A model state dictionary. + + Raises + ------ + InvalidStateError + If state is not a valid input for the DiagnosticComponent instance. + """ + if self._store_names is not None: + name_list = set(state.keys()).intersection(self._store_names) + cache_state = {name: state[name] for name in name_list} + else: + cache_state = state.copy() + + # raise an exception if the state has any empty string variables + for full_var_name in cache_state.keys(): + if len(full_var_name) == 0: + raise ValueError('The given state has an empty string as a variable name.') + + # replace cached variable names with their aliases + for longname, shortname in self._aliases.items(): + for full_var_name in tuple(cache_state.keys()): + # replace any string in the full variable name that matches longname + # example: if longname is "temperature", shortname is "T", and + # full_var_name is "temperature_tendency_from_radiation", the + # alias_name for the variable would be: "T_tendency_from_radiation" + if longname in full_var_name: + alias_name = full_var_name.replace(longname, shortname) + if len(alias_name) == 0: # raise exception if the alias is an empty str + errstr = 'Tried to alias variable "{}" to an empty string.\n' + \ + 'xarray will not allow empty strings as variable names.' + raise ValueError(errstr.format(full_var_name)) + cache_state[alias_name] = cache_state.pop(full_var_name) + + cache_state.pop('time') # stored as key, not needed in state dict + if state['time'] in self._cached_state_dict.keys(): + self._cached_state_dict[state['time']].update(cache_state) + else: + self._cached_state_dict[state['time']] = cache_state + if self._write_on_store: + self.write() + + @property + def _write_mode(self): + if not os.path.isfile(self._filename): + return 'w' + else: + return 'a' + + def _ensure_cached_state_keys_compatible_with_dataset(self, dataset): + file_keys = list(dataset.variables.keys()) + if 'time' in file_keys: + file_keys.remove('time') + if len(file_keys) > 0: + self._ensure_cached_states_have_same_keys(file_keys) + else: + self._ensure_cached_states_have_same_keys() + + def _ensure_cached_states_have_same_keys(self, desired_keys=None): + """ + Ensures all states in self._cached_state_dict have the same keys. + If desired_keys is given, also ensure the keys are the same as + the ones in desired_keys. + + Raises + ------ + InvalidStateError + If the cached states do not meet the requirements. + """ + if len(self._cached_state_dict) == 0: + return # trivially true + if desired_keys is not None: + reference_keys = desired_keys + else: + reference_state = tuple(self._cached_state_dict.values())[0] + reference_keys = reference_state.keys() + for state in self._cached_state_dict.values(): + if not same_list(list(state.keys()), list(reference_keys)): + raise InvalidStateError( + 'NetCDFMonitor was passed a different set of ' + 'quantities for different times: {} vs. {}'.format( + list(reference_keys), list(state.keys()))) + + def _get_ordered_times_and_states(self): + """Returns the items in self._cached_state_dict, sorted by time.""" + return zip(*sorted(self._cached_state_dict.items(), key=lambda x: x[0])) + + def write(self): + """ + Write all cached states to the NetCDF file, and clear the cache. + This will append to any existing NetCDF file. + + Raises + ------ + InvalidStateError + If cached states do not all have the same quantities + as every other cached and written state. + """ + with nc4.Dataset(self._filename, self._write_mode) as dataset: + self._ensure_cached_state_keys_compatible_with_dataset(dataset) + time_list, state_list = self._get_ordered_times_and_states() + self._ensure_time_exists(dataset, time_list[0]) + it_start = dataset.dimensions['time'].size + it_end = it_start + len(time_list) + append_times_to_dataset(time_list, dataset, self._time_units) + all_states = combine_states(state_list) + for name, value in all_states.items(): + ensure_variable_exists(dataset, name, value) + dataset.variables[name][ + it_start:it_end, :] = value.values[:, :] + self._cached_state_dict = {} + + def _ensure_time_exists(self, dataset, possible_reference_time): + """Ensure an unlimited time dimension relevant to this monitor + exists in the NetCDF4 dataset, and create it if it does not.""" + ensure_dimension_exists(dataset, 'time', None) + if 'time' not in dataset.variables: + dataset.createVariable('time', np.int64, ('time',)) + if isinstance(possible_reference_time, timedelta): + dataset.variables['time'].setncattr( + 'units', self._time_units) + else: # assume datetime + dataset.variables['time'].setncattr( + 'units', '{} since {}'.format( + self._time_units, possible_reference_time)) + dataset.variables['time'].setncattr( + 'calendar', 'proleptic_gregorian') class RestartMonitor(Monitor): @@ -218,6 +208,9 @@ class RestartMonitor(Monitor): """ def __init__(self, filename): + if nc4 is None: + raise DependencyError( + 'netCDF4-python must be installed to use RestartMonitor') self._filename = filename def store(self, state): From b2052ff6704624acb274aed9703778f203b5368a Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 15 Aug 2018 11:10:32 -0700 Subject: [PATCH 95/98] flake8 --- sympl/_components/netcdf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sympl/_components/netcdf.py b/sympl/_components/netcdf.py index 23ef016..cc87b6f 100644 --- a/sympl/_components/netcdf.py +++ b/sympl/_components/netcdf.py @@ -15,6 +15,7 @@ except ImportError: nc4 = None + class NetCDFMonitor(Monitor): """A Monitor which caches stored states and then writes them to a NetCDF file when requested.""" From 1dad2f48946e3109f98fafabe8fcbdc0a0e84703 Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 15 Aug 2018 13:10:15 -0700 Subject: [PATCH 96/98] Fix for UpdateFrequencyWrapper passing on timestep to TendencyComponent objects --- sympl/_core/wrappers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sympl/_core/wrappers.py b/sympl/_core/wrappers.py index 5ff57a5..63bb902 100644 --- a/sympl/_core/wrappers.py +++ b/sympl/_core/wrappers.py @@ -243,7 +243,10 @@ def __call__(self, state, timestep=None, **kwargs): (state['time'] >= self._last_update_time + self._update_timedelta)): if timestep is not None: - self._cached_output = self.component(state, timestep, **kwargs) + try: + self._cached_output = self.component(state, timestep, **kwargs) + except TypeError: + self._cached_output = self.component(state, **kwargs) else: self._cached_output = self.component(state, **kwargs) self._last_update_time = state['time'] From b0cbb59f9f1ba09a4a63f8237db1c306b18aa99c Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Wed, 15 Aug 2018 13:35:15 -0700 Subject: [PATCH 97/98] Fixed UpdateFrequencyWrapper having output_properties=None as an attribute --- sympl/_components/basic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sympl/_components/basic.py b/sympl/_components/basic.py index 7897d67..c8251c2 100644 --- a/sympl/_components/basic.py +++ b/sympl/_components/basic.py @@ -433,5 +433,7 @@ def array_call(self, state, timestep): raise NotImplementedError() def __getattr__(self, item): - if item not in ('outputs', 'output_properties'): + if item in ('outputs', 'output_properties'): + raise AttributeError() + else: return getattr(self._implicit, item) From b5d5de56a52b94c40eb8986756af0ab8394d8d3e Mon Sep 17 00:00:00 2001 From: Jeremy McGibbon Date: Thu, 16 Aug 2018 10:44:04 -0700 Subject: [PATCH 98/98] Fixed base class __str__ to use properties dicts --- sympl/_core/base_components.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sympl/_core/base_components.py b/sympl/_core/base_components.py index 8131048..3945bba 100644 --- a/sympl/_core/base_components.py +++ b/sympl/_core/base_components.py @@ -941,7 +941,8 @@ def __str__(self): ' inputs: {}\n' ' tendencies: {}\n' ' diagnostics: {}'.format( - self.__class__, self.inputs, self.tendencies, self.diagnostics) + self.__class__, self.input_properties.keys(), + self.tendency_properties.keys(), self.diagnostic_properties.keys()) ) def __repr__(self): @@ -1147,7 +1148,8 @@ def __str__(self): 'instance of {}(DiagnosticComponent)\n' ' inputs: {}\n' ' diagnostics: {}'.format( - self.__class__, self.inputs, self.diagnostics) + self.__class__, self.input_properties.keys(), + self.diagnostic_properties.keys()) ) def __repr__(self):