From d24fb96071cda6b64d3b4070cab605d8ad485541 Mon Sep 17 00:00:00 2001 From: lrennels Date: Tue, 26 Jan 2021 19:22:06 -0800 Subject: [PATCH 01/32] Play with propagate_time! function options --- src/core/build.jl | 2 +- src/core/defs.jl | 32 ++++++++++++++++++++++++++------ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/core/build.jl b/src/core/build.jl index 8bc9715fc..2c3e795c2 100644 --- a/src/core/build.jl +++ b/src/core/build.jl @@ -329,7 +329,7 @@ function _build(md::ModelDef) t = dimension(md, :time) time_bounds = (firstindex(t), lastindex(t)) - propagate_time!(md, t) + propagate_time!(md, t) ci = _build(md, vdict, pdict, time_bounds) mi = ModelInstance(ci, md) diff --git a/src/core/defs.jl b/src/core/defs.jl index bc7c9fa71..7fde8b129 100644 --- a/src/core/defs.jl +++ b/src/core/defs.jl @@ -824,18 +824,32 @@ function _insert_comp!(obj::AbstractCompositeComponentDef, comp_def::AbstractCom end """ - propagate_time!(obj::AbstractComponentDef, t::Dimension) + propagate_time!(obj::AbstractComponentDef, t::Dimension, first::NothingInt=nothing, last::NothingInt=nothing) Propagate a time dimension down through the comp def tree. """ -function propagate_time!(obj::AbstractComponentDef, t::Dimension) +function propagate_time!(obj::AbstractComponentDef, t::Dimension; first::NothingInt=nothing, last::NothingInt=nothing) set_dimension!(obj, :time, t) - obj.first = firstindex(t) - obj.last = lastindex(t) + # set first + parent_time_keys = [keys(t)...] + if isnothing(first) + obj.first = firstindex(t) + else + i = findfirst(isequal(first), parent_time_keys) + isnothing(i) ? error("The given first index must exist within the parent's time dimension.") : obj.first = first + end + + # set last + if isnothing(last) + obj.last = lastindex(t) + else + i = findfirst(isequal(last), parent_time_keys) + isnothing(i) ? error("The given last index must exist within the parent's time dimension.") : obj.first = first + end for c in compdefs(obj) # N.B. compdefs returns empty list for leaf nodes - propagate_time!(c, t) + propagate_time!(c, t, first = first, last = last) end end @@ -844,6 +858,8 @@ end obj::AbstractCompositeComponentDef, comp_def::AbstractComponentDef, comp_name::Symbol=comp_def.comp_id.comp_name; + first::NothingInt=nothing, + last::NothingInt=nothing, before::NothingSymbol=nothing, after::NothingSymbol=nothing, rename::NothingPairList=nothing @@ -857,6 +873,8 @@ argument `rename` can be a list of pairs indicating `original_name => imported_n function add_comp!(obj::AbstractCompositeComponentDef, comp_def::AbstractComponentDef, comp_name::Symbol=comp_def.comp_id.comp_name; + first::NothingInt=nothing, + last::NothingInt=nothing, before::NothingSymbol=nothing, after::NothingSymbol=nothing, rename::NothingPairList=nothing) # TBD: rename is not yet implemented @@ -871,7 +889,7 @@ function add_comp!(obj::AbstractCompositeComponentDef, # check time constraints if the time dimension has been set if has_dim(obj, :time) # error("Cannot add component to composite without first setting time dimension.") - propagate_time!(comp_def, dimension(obj, :time)) + propagate_time!(comp_def, dimension(obj, :time), first=first, last=last) end # Copy the original so we don't step on other uses of this comp @@ -891,6 +909,8 @@ end obj::AbstractCompositeComponentDef, comp_id::ComponentId, comp_name::Symbol=comp_id.comp_name; + first::NothingInt=nothing, + last::NothingInt=nothing, before::NothingSymbol=nothing, after::NothingSymbol=nothing, rename::NothingPairList=nothing From d759e19b6a07e795985a0e1a177fd15418762eed Mon Sep 17 00:00:00 2001 From: lrennels Date: Thu, 28 Jan 2021 01:11:45 -0800 Subject: [PATCH 02/32] More work on first and last; stuck on changing dims --- docs/src/ref/ref_structures_definitions.md | 2 + src/core/build.jl | 2 +- src/core/connections.jl | 13 +-- src/core/defs.jl | 104 +++++++-------------- src/core/dimensions.jl | 1 - src/core/types/defs.jl | 3 + wip/temp.jl | 76 +++++++++++++++ 7 files changed, 118 insertions(+), 83 deletions(-) create mode 100644 wip/temp.jl diff --git a/docs/src/ref/ref_structures_definitions.md b/docs/src/ref/ref_structures_definitions.md index 0c1bab5c5..a23a9cdce 100644 --- a/docs/src/ref/ref_structures_definitions.md +++ b/docs/src/ref/ref_structures_definitions.md @@ -23,6 +23,8 @@ dim_dict::OrderedDict{Symbol, Union{Nothing, Dimension}} namespace::OrderedDict{Symbol, Any} first::Union{Nothing, Int} last::Union{Nothing, Int} +first_free::Bool +last_free::Bool is_uniform::Bool ``` The namespace of a leaf component can hold `ParameterDef`s and `VariableDef`s, both which are subclasses of `DatumDef` (see below for more details on these types). diff --git a/src/core/build.jl b/src/core/build.jl index 2c3e795c2..86721bbb7 100644 --- a/src/core/build.jl +++ b/src/core/build.jl @@ -329,7 +329,7 @@ function _build(md::ModelDef) t = dimension(md, :time) time_bounds = (firstindex(t), lastindex(t)) - propagate_time!(md, t) + propagate_time!(md, t) # why do we need this here? ci = _build(md, vdict, pdict, time_bounds) mi = ModelInstance(ci, md) diff --git a/src/core/connections.jl b/src/core/connections.jl index b19bda5b0..6bc95f002 100644 --- a/src/core/connections.jl +++ b/src/core/connections.jl @@ -527,6 +527,7 @@ function _update_scalar_param!(param::ScalarModelParameter, name, value) end function _update_array_param!(obj::AbstractCompositeComponentDef, name, value, update_timesteps, raise_error) + # Get original parameter param = external_param(obj, name) @@ -542,15 +543,9 @@ function _update_array_param!(obj::AbstractCompositeComponentDef, name, value, u end end - # Check size of provided parameter - if update_timesteps && param.values isa TimestepArray - expected_size = ([length(dim_keys(obj, d)) for d in dim_names(param)]...,) - else - expected_size = size(param.values) - end - if size(value) != expected_size - error("Cannot update parameter $name; expected array of size $expected_size but got array of size $(size(value)).") - end + # Check if the parameter dimensions match the model dimensions + curr_dims_size = ([length(dim_keys(obj, d)) for d in dim_names(param)]...,) # the size of the dimension (might have been updated) + size(value) != curr_dims_size ? error("Cannot update parameter $name; expected array of size $curr_dims_size but got array of size $(size(value)).") : nothing if update_timesteps if param.values isa TimestepArray diff --git a/src/core/defs.jl b/src/core/defs.jl index 7fde8b129..35cc1b880 100644 --- a/src/core/defs.jl +++ b/src/core/defs.jl @@ -264,65 +264,6 @@ function datum_size(obj::AbstractCompositeComponentDef, comp_def::ComponentDef, return datum_size end -""" - _check_run_period(obj::AbstractComponentDef, first, last) - -Raise an error if the component has an earlier start than `first` or a later finish than -`last`. Values of `nothing` are not checked. Composites recurse to check sub-components. -""" -function _check_run_period(obj::AbstractComponentDef, new_first, new_last) - # @info "_check_run_period($(obj.comp_id), $(printable(new_first)), $(printable(new_last))" - old_first = first_period(obj) - old_last = last_period(obj) - - if new_first !== nothing && old_first !== nothing && new_first < old_first - error("Attempted to set first period of $(obj.comp_id) to an earlier period ($new_first) than component indicates ($old_first)") - end - - if new_last !== nothing && old_last !== nothing && new_last > old_last - error("Attempted to set last period of $(obj.comp_id) to a later period ($new_last) than component indicates ($old_last)") - end - - # N.B. compdefs() returns an empty list for leaf ComponentDefs - for subcomp in compdefs(obj) - _check_run_period(subcomp, new_first, new_last) - end - - nothing -end - -""" - _set_run_period!(obj::AbstractComponentDef, first, last) - -Allows user to change the bounds on a AbstractComponentDef's time dimension. -An error is raised if the new time bounds are outside those of any -subcomponent, recursively. -""" -function _set_run_period!(obj::AbstractComponentDef, first, last) - # We've disabled `first` and `last` args to add_comp!, so we don't test bounds - # _check_run_period(obj, first, last) - - first_per = first_period(obj) - last_per = last_period(obj) - changed = false - - if first !== nothing - obj.first = first - changed = true - end - - if last !== nothing - obj.last = last - changed = true - end - - if changed - dirty!(obj) - end - - nothing -end - # helper functions used to determine if the provided time values are # a uniform range. @@ -824,32 +765,42 @@ function _insert_comp!(obj::AbstractCompositeComponentDef, comp_def::AbstractCom end """ - propagate_time!(obj::AbstractComponentDef, t::Dimension, first::NothingInt=nothing, last::NothingInt=nothing) + propagate_time!(obj::AbstractComponentDef, t::Dimension; first::NothingInt=nothing, last::NothingInt=nothing) -Propagate a time dimension down through the comp def tree. +Propagate a time dimension down through the comp def tree. If first and last +keyword arguments are included as integers, then the object's first_free and/or +last_free flags are set to false respectively, these first and last are propagated +through, and they will not vary freely with the model. """ function propagate_time!(obj::AbstractComponentDef, t::Dimension; first::NothingInt=nothing, last::NothingInt=nothing) + set_dimension!(obj, :time, t) - + # set first parent_time_keys = [keys(t)...] - if isnothing(first) + if isnothing(first) && obj.first_free obj.first = firstindex(t) + elseif isnothing(first) && !obj.first_free + # do nothing in this case, must leave first alone it is locked else + obj.first_free = false i = findfirst(isequal(first), parent_time_keys) isnothing(i) ? error("The given first index must exist within the parent's time dimension.") : obj.first = first end # set last - if isnothing(last) + if isnothing(last) && obj.last_free obj.last = lastindex(t) + elseif isnothing(last) && !obj.last_free + # do nothing in this case, must leave last alone it is locked else + obj.last_free = false i = findfirst(isequal(last), parent_time_keys) - isnothing(i) ? error("The given last index must exist within the parent's time dimension.") : obj.first = first + isnothing(i) ? error("The given last index must exist within the parent's time dimension.") : obj.last = last end for c in compdefs(obj) # N.B. compdefs returns empty list for leaf nodes - propagate_time!(c, t, first = first, last = last) + propagate_time!(c, t, first=first, last=last) end end @@ -860,6 +811,8 @@ end comp_name::Symbol=comp_def.comp_id.comp_name; first::NothingInt=nothing, last::NothingInt=nothing, + first_free::Bool=true, + last_free::Bool=true, before::NothingSymbol=nothing, after::NothingSymbol=nothing, rename::NothingPairList=nothing @@ -875,6 +828,8 @@ function add_comp!(obj::AbstractCompositeComponentDef, comp_name::Symbol=comp_def.comp_id.comp_name; first::NothingInt=nothing, last::NothingInt=nothing, + first_free::Bool=true, + last_free::Bool=true, before::NothingSymbol=nothing, after::NothingSymbol=nothing, rename::NothingPairList=nothing) # TBD: rename is not yet implemented @@ -886,17 +841,20 @@ function add_comp!(obj::AbstractCompositeComponentDef, error("Cannot specify both 'before' and 'after' parameters") end - # check time constraints if the time dimension has been set - if has_dim(obj, :time) - # error("Cannot add component to composite without first setting time dimension.") - propagate_time!(comp_def, dimension(obj, :time), first=first, last=last) - end - # Copy the original so we don't step on other uses of this comp comp_def = deepcopy(comp_def) comp_def.name = comp_name parent!(comp_def, obj) + # Handle time dimension for the copy, leave the time unset for the original + # component template + if has_dim(obj, :time) + propagate_time!(comp_def, dimension(obj, :time), first=first, last=last) + else + # can't error or composites won't work + # error("Cannot add component to composite without first setting time dimension.") + end + _add_anonymous_dims!(obj, comp_def) _insert_comp!(obj, comp_def, before=before, after=after) @@ -911,6 +869,8 @@ end comp_name::Symbol=comp_id.comp_name; first::NothingInt=nothing, last::NothingInt=nothing, + first_free::Bool=true, + last_free::Bool=true, before::NothingSymbol=nothing, after::NothingSymbol=nothing, rename::NothingPairList=nothing diff --git a/src/core/dimensions.jl b/src/core/dimensions.jl index cab527b6e..bac53bad0 100644 --- a/src/core/dimensions.jl +++ b/src/core/dimensions.jl @@ -94,7 +94,6 @@ function set_dimension!(ccd::AbstractCompositeComponentDef, name::Symbol, keys:: dim = Dimension(keys) if name == :time - _set_run_period!(ccd, keys[1], keys[end]) propagate_time!(ccd, dim) set_uniform!(ccd, isuniform(keys)) end diff --git a/src/core/types/defs.jl b/src/core/types/defs.jl index efb029e2b..feb80c072 100644 --- a/src/core/types/defs.jl +++ b/src/core/types/defs.jl @@ -42,6 +42,8 @@ end namespace::OrderedDict{Symbol, Any} first::Union{Nothing, Int} last::Union{Nothing, Int} + first_free::Bool + last_free::Bool is_uniform::Bool # Store a reference to the AbstractCompositeComponent that contains this comp def. @@ -73,6 +75,7 @@ end self.dim_dict = OrderedDict{Symbol, Union{Nothing, Dimension}}() self.namespace = OrderedDict{Symbol, Any}() self.first = self.last = nothing + self.first_free = self.last_free = true self.is_uniform = true self.parent = nothing return self diff --git a/wip/temp.jl b/wip/temp.jl new file mode 100644 index 000000000..4fd88c8d2 --- /dev/null +++ b/wip/temp.jl @@ -0,0 +1,76 @@ +using Mimi # start by importing the Mimi package to your space + +@defcomp grosseconomy begin + YGROSS = Variable(index=[time]) # Gross output + K = Variable(index=[time]) # Capital + l = Parameter(index=[time]) # Labor + tfp = Parameter(index=[time]) # Total factor productivity + s = Parameter(index=[time]) # Savings rate + depk = Parameter() # Depreciation rate on capital - Note that it has no time index + k0 = Parameter() # Initial level of capital + share = Parameter() # Capital share + + function run_timestep(p, v, d, t) + # Define an equation for K + if is_first(t) + # Note the use of v. and p. to distinguish between variables and parameters + v.K[t] = p.k0 + else + v.K[t] = (1 - p.depk)^5 * v.K[t-1] + v.YGROSS[t-1] * p.s[t-1] * 5 + end + + # Define an equation for YGROSS + v.YGROSS[t] = p.tfp[t] * v.K[t]^p.share * p.l[t]^(1-p.share) + end +end + +@defcomp emissions begin + E = Variable(index=[time]) # Total greenhouse gas emissions + sigma = Parameter(index=[time]) # Emissions output ratio + YGROSS = Parameter(index=[time]) # Gross output - Note that YGROSS is now a parameter + + function run_timestep(p, v, d, t) + + # Define an equation for E + v.E[t] = p.YGROSS[t] * p.sigma[t] # Note the p. in front of YGROSS + end +end + + +function construct_model() + m = Model() + + set_dimension!(m, :time, collect(2015:5:2110)) + + # Order matters here. If the emissions component were defined first, the model would not run. + add_comp!(m, grosseconomy) + add_comp!(m, emissions, first = 2020, last = 2105) + + # Set parameters for the grosseconomy component + set_param!(m, :grosseconomy, :l, [(1. + 0.015)^t *6404 for t in 1:20]) + set_param!(m, :grosseconomy, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20]) + set_param!(m, :grosseconomy, :s, ones(20).* 0.22) + set_param!(m, :grosseconomy, :depk, 0.1) + set_param!(m, :grosseconomy, :k0, 130.) + set_param!(m, :grosseconomy, :share, 0.3) + + # Set parameters for the emissions component + set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:18]) + connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) + # Note that connect_param! was used here. + + return m + +end #end function + +m = construct_model() +run(m) +set_dimension!(m, :time, collect(2015:5:2115)) + +# Set parameters for the grosseconomy component +update_param!(m, :l, [(1. + 0.015)^t *6404 for t in 1:20],update_timesteps=true) +update_param!(m, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20],update_timesteps=true) + +# Set parameters for the emissions component +update_param!(m, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20], update_timesteps=true) +run(m) \ No newline at end of file From d20a7201f318ca481ebce4d9c9f50bd7e9993e59 Mon Sep 17 00:00:00 2001 From: lrennels Date: Thu, 28 Jan 2021 13:01:08 -0800 Subject: [PATCH 03/32] Add tests --- src/core/connections.jl | 11 ++++++++--- test/test_parametertypes.jl | 22 ++++++++++------------ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/core/connections.jl b/src/core/connections.jl index 6bc95f002..74e45d5ac 100644 --- a/src/core/connections.jl +++ b/src/core/connections.jl @@ -543,9 +543,14 @@ function _update_array_param!(obj::AbstractCompositeComponentDef, name, value, u end end - # Check if the parameter dimensions match the model dimensions - curr_dims_size = ([length(dim_keys(obj, d)) for d in dim_names(param)]...,) # the size of the dimension (might have been updated) - size(value) != curr_dims_size ? error("Cannot update parameter $name; expected array of size $curr_dims_size but got array of size $(size(value)).") : nothing + # Check if the parameter dimensions match the model dimensions. Note that we + # previously checked if parameter dimensions matched the dimensions of the + # parameter they were to replace, but given dimensions of a model can be changed, + # we now choose to enforce that the new dimensions match the current model state, + # whatever that is. + + expected_size = ([length(dim_keys(obj, d)) for d in dim_names(param)]...,) + size(value) != expected_size ? error("Cannot update parameter $name; expected array of size $expected_size but got array of size $(size(value)).") : nothing if update_timesteps if param.values isa TimestepArray diff --git a/test/test_parametertypes.jl b/test/test_parametertypes.jl index e99351e2e..9beb8be11 100644 --- a/test/test_parametertypes.jl +++ b/test/test_parametertypes.jl @@ -140,17 +140,18 @@ set_param!(m, :MyComp2, :x, [1, 2, 3]) # Can't move last beyond last for a component # @test_throws ErrorException set_dimension!(m, :time, 2001:2003) -set_dimension!(m, :time, 2001:2002) +set_dimension!(m, :time, 2000:2001) -update_param!(m, :x, [4, 5, 6], update_timesteps = false) +@test_throws ErrorException update_param!(m, :x, [4, 5, 6], update_timesteps = false) # wrong size +update_param!(m, :x, [4, 5]; update_timesteps = false) x = external_param(m.md, :x) @test x.values isa Mimi.TimestepArray{Mimi.FixedTimestep{2000, 1, LAST} where LAST, Union{Missing,Float64}, 1} -@test x.values.data == [4., 5., 6.] -# TBD: this fails, but I'm not sure how it's supposed to behave. It says: -# (ERROR: BoundsError: attempt to access 3-element Array{Float64,1} at index [4]) -# run(m) -# @test m[:MyComp2, :y][1] == 5 # 2001 -# @test m[:MyComp2, :y][2] == 6 # 2002 +@test x.values.data == [4., 5.] +run(m) +@test m[:MyComp2, :y][1] == 4 # 2000 +@test m[:MyComp2, :y][2] == 5 # 2001 + +set_dimension!(m, :time, 2001:2002) update_param!(m, :x, [2, 3], update_timesteps = true) x = external_param(m.md, :x) @@ -174,9 +175,7 @@ update_param!(m, :x, [4, 5, 6], update_timesteps = false) x = external_param(m.md, :x) @test x.values isa Mimi.TimestepArray{Mimi.VariableTimestep{(2000, 2005, 2020)}, Union{Missing,Float64}, 1} @test x.values.data == [4., 5., 6.] -#run(m) -#@test m[:MyComp2, :y][1] == 5 # 2005 -#@test m[:MyComp2, :y][2] == 6 # 2020 +@test_throws BoundsError run(m) # can't run the model due to timestep inconsistencies, it is in an invalid state update_param!(m, :x, [2, 3, 4], update_timesteps = true) x = external_param(m.md, :x) @@ -215,7 +214,6 @@ set_param!(m, :MyComp2, :x, [1, 2, 3]) set_dimension!(m, :time, 1999:2003) # length 5 -@test_throws ErrorException update_param!(m, :x, [2, 3, 4, 5, 6], update_timesteps = false) update_param!(m, :x, [2, 3, 4, 5, 6], update_timesteps = true) x = external_param(m.md, :x) @test x.values isa Mimi.TimestepArray{Mimi.FixedTimestep{1999, 1, LAST} where LAST, Union{Missing,Float64}, 1} From 6285be889d54a6a690189617599012d00d1eb1de Mon Sep 17 00:00:00 2001 From: lrennels Date: Thu, 28 Jan 2021 14:35:47 -0800 Subject: [PATCH 04/32] Remove update_timesteps keyword argument --- docs/src/howto/howto_4.md | 5 ++-- docs/src/howto/howto_5.md | 6 ++-- docs/src/tutorials/tutorial_3.md | 14 +++------- src/core/build.jl | 2 +- src/core/connections.jl | 47 ++++++++++++-------------------- src/core/model.jl | 18 ++++++------ test/test_components.jl | 2 +- test/test_parametertypes.jl | 27 ++++-------------- wip/temp.jl | 10 +++---- 9 files changed, 46 insertions(+), 85 deletions(-) diff --git a/docs/src/howto/howto_4.md b/docs/src/howto/howto_4.md index a72e35ae1..a14f6bf9d 100644 --- a/docs/src/howto/howto_4.md +++ b/docs/src/howto/howto_4.md @@ -101,11 +101,10 @@ In both of these cases, the parameter's values are stored of as an array (p1 is ## Updating an external parameter -When `set_param!` is called, it creates an external parameter by the name provided, and stores the provided scalar or array value. It is possible to later change the value associated with that parameter name using the functions described below. If the external parameter has a `:time` dimension, use the optional argument `update_timesteps=true` to indicate that the time keys (i.e., year labels) associated with the parameter should be updated in addition to updating the parameter values. +When `set_param!` is called, it creates an external parameter by the name provided, and stores the provided scalar or array value. It is possible to later change the value associated with that parameter name using the functions described below. ```julia -update_param!(m, :ParameterName, newvalues) # update values only -update_param!(m, :ParameterName, newvalues, update_timesteps=true) # also update time keys +update_param!(m, :ParameterName, newvalues) ``` Note: `newvalues` must be the same size and type (or be able to convert to the type) of the old values stored in that parameter. diff --git a/docs/src/howto/howto_5.md b/docs/src/howto/howto_5.md index 03740f864..c1848e9aa 100644 --- a/docs/src/howto/howto_5.md +++ b/docs/src/howto/howto_5.md @@ -96,11 +96,9 @@ The full API: To update an external parameter, use the functions `update_param!` and `udpate_params!` (previously known as `update_external_parameter` and `update_external_parameters`, respectively.) Their calling signatures are: -* `update_params!(md::ModelDef, parameters::Dict; update_timesteps = false)` +* `update_params!(md::ModelDef, parameters::Dict)` -* `update_param!(md::ModelDef, name::Symbol, value; update_timesteps = false)` - -For external parameters with a `:time` dimension, passing `update_timesteps=true` indicates that the time _keys_ (i.e., year labels) should also be updated in addition to updating the parameter values. +* `update_param!(md::ModelDef, name::Symbol, value)` #### Setting parameters with a dictionary diff --git a/docs/src/tutorials/tutorial_3.md b/docs/src/tutorials/tutorial_3.md index 1ce670bce..fd5ee12ea 100644 --- a/docs/src/tutorials/tutorial_3.md +++ b/docs/src/tutorials/tutorial_3.md @@ -22,15 +22,13 @@ Possible modifications range in complexity, from simply altering parameter value Several types of changes to models revolve around the parameters themselves, and may include updating the values of parameters and changing parameter connections without altering the elements of the components themselves or changing the general component structure of the model. The most useful functions of the common API in these cases are likely **[`update_param!`](@ref)/[`update_params!`](@ref), [`disconnect_param!`](@ref), and [`connect_param!`](@ref)**. For detail on these functions see the API reference guide, Reference Guide: The Mimi API. -When the original model calls [`set_param!`](@ref), Mimi creates an external parameter by the name provided, and stores the provided scalar or array value. The functions [`update_param!`](@ref) and [`update_params!`](@ref) allow you to change the value associated with this external parameter. Note that if the external parameter has a `:time` dimension, use the optional argument `update_timesteps=true` to indicate that the time keys (i.e., year labels) associated with the parameter should be updated in addition to updating the parameter values. +When the original model calls [`set_param!`](@ref), Mimi creates an external parameter by the name provided, and stores the provided scalar or array value. The functions [`update_param!`](@ref) and [`update_params!`](@ref) allow you to change the value associated with this external parameter. ```julia -update_param!(mymodel, :parametername, newvalues) # update values only - -update_param!(mymodel, :parametername, newvalues, update_timesteps=true) # also update time keys. Only necessary if the time dimension of the model has been changed. +update_param!(mymodel, :parametername, newvalues) ``` -In the code above, `newvalues` must be the same size and type (or be able to convert to the type) of the old values stored in that parameter. +In the code above, `newvalues` must be the same size and type (or be able to convert to the type) as required by the model dimensions set with `set_dimension!`. If you wish to alter connections within an existing model, [`disconnect_param!`](@ref) and [`connect_param!`](@ref) can be used in conjunction with each other to update the connections within the model, although this is more likely to be done as part of larger changes involving components themslves, as discussed in the next subsection. @@ -96,8 +94,6 @@ nyears = length(years) set_dimension!(m, :time, years) ``` -Now that you have changed the time dimension, you have a mismatch between the time labels attached to your parameters and the time labels used by the model. Thus, **you must update at least all parameters with a `:time`** dimension and use the explicit `update_timesteps=true` flag to get the time labels on the parameters to match those in the model. This is required even in cases where you do not want to change the parameter values themselves (see the forum question [here](https://forum.mimiframework.org/t/update-time-index/134/5)) for an in-depth explanation of this case. You may of course also update parameters without a `:time` dimension as desired. - Create a dictionary `params` with one entry `(k, v)` per external parameter by name `k` to value `v`. Each key `k` must be a symbol or convert to a symbol matching the name of an external parameter that already exists in the model definition. Part of this dictionary may look like: ```julia @@ -112,12 +108,10 @@ params[:S] = repeat([0.23], nyears) Now you simply update the parameters listen in `params` and re-run the model with ```julia -update_params!(m, params, update_timesteps=true) +update_params!(m, params) run(m) ``` -Note again that here we use the `update_timesteps` flag and set it to `true`, because since we have changed the time index we want the time labels on the parameters to change, not simply their values. - ## Component and Structural Modifications: The API Most model modifications will include not only parametric updates, but also structural changes and component modification, addition, replacement, and deletion along with the required re-wiring of parameters etc. The most useful functions of the common API, in these cases are likely **[`replace!`](@ref), [`add_comp!`](@ref)** along with **`delete!`** and the requisite functions for parameter setting and connecting. For detail on the public API functions look at the API reference. diff --git a/src/core/build.jl b/src/core/build.jl index 86721bbb7..e12aff802 100644 --- a/src/core/build.jl +++ b/src/core/build.jl @@ -329,7 +329,7 @@ function _build(md::ModelDef) t = dimension(md, :time) time_bounds = (firstindex(t), lastindex(t)) - propagate_time!(md, t) # why do we need this here? + propagate_time!(md, t) # this might not be needed, but is a final propagation to double check everything ci = _build(md, vdict, pdict, time_bounds) mi = ModelInstance(ci, md) diff --git a/src/core/connections.jl b/src/core/connections.jl index 74e45d5ac..d1155883e 100644 --- a/src/core/connections.jl +++ b/src/core/connections.jl @@ -470,15 +470,13 @@ function set_external_scalar_param!(obj::ModelDef, name::Symbol, value::Any) end """ - update_param!(obj::AbstractCompositeComponentDef, name::Symbol, value; update_timesteps = false) + update_param!(obj::AbstractCompositeComponentDef, name::Symbol, value) Update the `value` of an external model parameter in composite `obj`, referenced -by `name`. Optional boolean argument `update_timesteps` with default value -`false` indicates whether to update the time keys associated with the parameter -values to match the model's time index. +by `name`. """ -function update_param!(obj::AbstractCompositeComponentDef, name::Symbol, value; update_timesteps = false) - _update_param!(obj::AbstractCompositeComponentDef, name, value, update_timesteps; raise_error = true) +function update_param!(obj::AbstractCompositeComponentDef, name::Symbol, value) + _update_param!(obj::AbstractCompositeComponentDef, name, value) end function update_param!(mi::ModelInstance, name::Symbol, value) @@ -496,19 +494,16 @@ function update_param!(mi::ModelInstance, name::Symbol, value) end function _update_param!(obj::AbstractCompositeComponentDef, - name::Symbol, value, update_timesteps; raise_error = true) + name::Symbol, value) param = external_param(obj, name, missing_ok=true) if param === nothing error("Cannot update parameter; $name not found in composite's external parameters.") end if param isa ScalarModelParameter - if update_timesteps && raise_error - error("Cannot update timesteps; parameter $name is a scalar parameter.") - end _update_scalar_param!(param, name, value) else - _update_array_param!(obj, name, value, update_timesteps, raise_error) + _update_array_param!(obj, name, value) end dirty!(obj) @@ -526,7 +521,7 @@ function _update_scalar_param!(param::ScalarModelParameter, name, value) nothing end -function _update_array_param!(obj::AbstractCompositeComponentDef, name, value, update_timesteps, raise_error) +function _update_array_param!(obj::AbstractCompositeComponentDef, name, value) # Get original parameter param = external_param(obj, name) @@ -552,44 +547,38 @@ function _update_array_param!(obj::AbstractCompositeComponentDef, name, value, u expected_size = ([length(dim_keys(obj, d)) for d in dim_names(param)]...,) size(value) != expected_size ? error("Cannot update parameter $name; expected array of size $expected_size but got array of size $(size(value)).") : nothing - if update_timesteps - if param.values isa TimestepArray + # check if updating timestep labels is necessary + if param.values isa TimestepArray + time_label_change = time_labels(param.values) != dim_keys(obj, :time) + if time_label_change T = eltype(value) N = length(size(value)) ti = get_time_index_position(param) new_timestep_array = get_timestep_array(obj, T, N, ti, value) set_external_param!(obj, name, ArrayModelParameter(new_timestep_array, dim_names(param))) - - elseif raise_error - error("Cannot update timesteps; parameter $name is not a TimestepArray.") else - param.values = value - end - else - if param.values isa TimestepArray param.values.data = value - else - param.values = value end + else + param.values = value end + dirty!(obj) nothing end """ - update_params!(obj::AbstractCompositeComponentDef, parameters::Dict{T, Any}; - update_timesteps = false) where T + update_params!(obj::AbstractCompositeComponentDef, parameters::Dict{T, Any}) where T For each (k, v) in the provided `parameters` dictionary, `update_param!` -is called to update the external parameter by name k to value v, with optional -Boolean argument update_timesteps. Each key k must be a symbol or convert to a +is called to update the external parameter by name k to value v. Each key k must be a symbol or convert to a symbol matching the name of an external parameter that already exists in the component definition. """ -function update_params!(obj::AbstractCompositeComponentDef, parameters::Dict; update_timesteps = false) +function update_params!(obj::AbstractCompositeComponentDef, parameters::Dict) parameters = Dict(Symbol(k) => v for (k, v) in parameters) for (param_name, value) in parameters - _update_param!(obj, param_name, value, update_timesteps; raise_error = false) + _update_param!(obj, param_name, value) end nothing end diff --git a/src/core/model.jl b/src/core/model.jl index 92df8f87a..5e3141da5 100644 --- a/src/core/model.jl +++ b/src/core/model.jl @@ -99,25 +99,23 @@ function set_leftover_params!(m::Model, parameters::Dict{T, Any}) where T end """ - update_param!(m::Model, name::Symbol, value; update_timesteps = false) + update_param!(m::Model, name::Symbol, value) Update the `value` of an external model parameter in model `m`, referenced by -`name`. Optional boolean argument `update_timesteps` with default value `false` -indicates whether to update the time keys associated with the parameter values -to match the model's time index. +`name`. """ -@delegate update_param!(m::Model, name::Symbol, value; update_timesteps = false) => md +@delegate update_param!(m::Model, name::Symbol, value) => md """ - update_params!(m::Model, parameters::Dict{T, Any}; update_timesteps = false) where T + update_params!(m::Model, parameters::Dict{T, Any}) where T For each (k, v) in the provided `parameters` dictionary, `update_param!`` -is called to update the external parameter by name k to value v, with optional -Boolean argument update_timesteps. Each key k must be a symbol or convert to a -symbol matching the name of an external parameter that already exists in the +is called to update the external parameter by name k to value v. Each key k +must be a symbol or convert to a symbol matching the name of an external parameter t +hat already exists in the model definition. """ -@delegate update_params!(m::Model, parameters::Dict; update_timesteps = false) => md +@delegate update_params!(m::Model, parameters::Dict) => md """ add_comp!( diff --git a/test/test_components.jl b/test/test_components.jl index 8b043645c..30260bfee 100644 --- a/test/test_components.jl +++ b/test/test_components.jl @@ -130,7 +130,7 @@ cd = compdef(m.md, :C) # Get the component definition in the model #@test cd.first === nothing # First and last values should still be nothing #@test cd.last === nothing -update_param!(m, :par1, zeros(16); update_timesteps=true) +update_param!(m, :par1, zeros(16)) Mimi.build!(m) # Build the model ci = compinstance(m, :C) # Get the component instance @test ci.first == 2005 # The component instance's first and last values should match the model's index diff --git a/test/test_parametertypes.jl b/test/test_parametertypes.jl index 9beb8be11..ea24912b6 100644 --- a/test/test_parametertypes.jl +++ b/test/test_parametertypes.jl @@ -142,18 +142,9 @@ set_param!(m, :MyComp2, :x, [1, 2, 3]) set_dimension!(m, :time, 2000:2001) -@test_throws ErrorException update_param!(m, :x, [4, 5, 6], update_timesteps = false) # wrong size -update_param!(m, :x, [4, 5]; update_timesteps = false) -x = external_param(m.md, :x) -@test x.values isa Mimi.TimestepArray{Mimi.FixedTimestep{2000, 1, LAST} where LAST, Union{Missing,Float64}, 1} -@test x.values.data == [4., 5.] -run(m) -@test m[:MyComp2, :y][1] == 4 # 2000 -@test m[:MyComp2, :y][2] == 5 # 2001 - set_dimension!(m, :time, 2001:2002) -update_param!(m, :x, [2, 3], update_timesteps = true) +update_param!(m, :x, [2, 3]) x = external_param(m.md, :x) @test x.values isa Mimi.TimestepArray{Mimi.FixedTimestep{2001, 1, LAST} where LAST, Union{Missing,Float64}, 1} @test x.values.data == [2., 3.] @@ -171,13 +162,7 @@ set_param!(m, :MyComp2, :x, [1, 2, 3]) set_dimension!(m, :time, [2005, 2020, 2050]) -update_param!(m, :x, [4, 5, 6], update_timesteps = false) -x = external_param(m.md, :x) -@test x.values isa Mimi.TimestepArray{Mimi.VariableTimestep{(2000, 2005, 2020)}, Union{Missing,Float64}, 1} -@test x.values.data == [4., 5., 6.] -@test_throws BoundsError run(m) # can't run the model due to timestep inconsistencies, it is in an invalid state - -update_param!(m, :x, [2, 3, 4], update_timesteps = true) +update_param!(m, :x, [2, 3, 4]) x = external_param(m.md, :x) @test x.values isa Mimi.TimestepArray{Mimi.VariableTimestep{(2005, 2020, 2050)}, Union{Missing,Float64}, 1} @test x.values.data == [2., 3., 4.] @@ -195,7 +180,7 @@ set_param!(m, :MyComp2, :x, [1, 2, 3]) set_dimension!(m, :time, [2005, 2020, 2050]) -update_params!(m, Dict(:x=>[2, 3, 4]), update_timesteps = true) +update_params!(m, Dict(:x=>[2, 3, 4])) x = external_param(m.md, :x) @test x.values isa Mimi.TimestepArray{Mimi.VariableTimestep{(2005, 2020, 2050)}, Union{Missing,Float64}, 1} @test x.values.data == [2., 3., 4.] @@ -214,7 +199,7 @@ set_param!(m, :MyComp2, :x, [1, 2, 3]) set_dimension!(m, :time, 1999:2003) # length 5 -update_param!(m, :x, [2, 3, 4, 5, 6], update_timesteps = true) +update_param!(m, :x, [2, 3, 4, 5, 6]) x = external_param(m.md, :x) @test x.values isa Mimi.TimestepArray{Mimi.FixedTimestep{1999, 1, LAST} where LAST, Union{Missing,Float64}, 1} @test x.values.data == [2., 3., 4., 5., 6.] @@ -240,17 +225,15 @@ set_param!(m, :MyComp3, :y, [10, 20]) set_param!(m, :MyComp3, :z, 0) @test_throws ErrorException update_param!(m, :x, [1, 2, 3, 4]) # Will throw an error because size -@test_throws ErrorException update_param!(m, :y, [10, 15], update_timesteps=true) # Not a timestep array update_param!(m, :y, [10, 15]) @test external_param(m.md, :y).values == [10., 15.] -@test_throws ErrorException update_param!(m, :z, 1, update_timesteps=true) # Scalar parameter update_param!(m, :z, 1) @test external_param(m.md, :z).value == 1 # Reset the time dimensions set_dimension!(m, :time, 2005:2007) -update_params!(m, Dict(:x=>[3,4,5], :y=>[10,20], :z=>0), update_timesteps=true) # Won't error when updating from a dictionary +update_params!(m, Dict(:x=>[3,4,5], :y=>[10,20], :z=>0)) # Won't error when updating from a dictionary @test external_param(m.md, :x).values isa Mimi.TimestepArray{Mimi.FixedTimestep{2005,1},Union{Missing,Float64},1} @test external_param(m.md, :x).values.data == [3.,4.,5.] diff --git a/wip/temp.jl b/wip/temp.jl index 4fd88c8d2..236491eea 100644 --- a/wip/temp.jl +++ b/wip/temp.jl @@ -44,7 +44,7 @@ function construct_model() # Order matters here. If the emissions component were defined first, the model would not run. add_comp!(m, grosseconomy) - add_comp!(m, emissions, first = 2020, last = 2105) + add_comp!(m, emissions, first = 2020) # Set parameters for the grosseconomy component set_param!(m, :grosseconomy, :l, [(1. + 0.015)^t *6404 for t in 1:20]) @@ -55,7 +55,7 @@ function construct_model() set_param!(m, :grosseconomy, :share, 0.3) # Set parameters for the emissions component - set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:18]) + set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:19]) connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) # Note that connect_param! was used here. @@ -68,9 +68,9 @@ run(m) set_dimension!(m, :time, collect(2015:5:2115)) # Set parameters for the grosseconomy component -update_param!(m, :l, [(1. + 0.015)^t *6404 for t in 1:20],update_timesteps=true) -update_param!(m, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20],update_timesteps=true) +update_param!(m, :l, [(1. + 0.015)^t *6404 for t in 1:20]) +update_param!(m, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20]) # Set parameters for the emissions component -update_param!(m, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20], update_timesteps=true) +update_param!(m, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) run(m) \ No newline at end of file From 0e2c335f8f6ced008ce619a8d76adf9ec3538a74 Mon Sep 17 00:00:00 2001 From: lrennels Date: Thu, 28 Jan 2021 14:59:29 -0800 Subject: [PATCH 05/32] Fix documentation --- docs/src/howto/howto_4.md | 2 +- docs/src/howto/howto_5.md | 8 +++++--- docs/src/tutorials/tutorial_3.md | 4 +++- src/core/connections.jl | 6 ++++++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/src/howto/howto_4.md b/docs/src/howto/howto_4.md index a14f6bf9d..8cf62b26c 100644 --- a/docs/src/howto/howto_4.md +++ b/docs/src/howto/howto_4.md @@ -107,7 +107,7 @@ When `set_param!` is called, it creates an external parameter by the name provid update_param!(m, :ParameterName, newvalues) ``` -Note: `newvalues` must be the same size and type (or be able to convert to the type) of the old values stored in that parameter. +Note here that `newvalues` must be the same type (or be able to convert to the type) of the old values stored in that parameter, and the same size as the model dimensions indicate. Also note that it if you have updated the time dimension of the model with `set_dimension!(m, :time, values)` you will need to update all parameters with a `:time` dimension, **even if the values have not changed**, so that the model can update the underlying time labels attached to the parameters. #### Setting parameters with a dictionary diff --git a/docs/src/howto/howto_5.md b/docs/src/howto/howto_5.md index c1848e9aa..7e61ca7c2 100644 --- a/docs/src/howto/howto_5.md +++ b/docs/src/howto/howto_5.md @@ -50,7 +50,7 @@ In an effort to standardize the function naming protocol within Mimi, and to str |`addcomponent!` |`add_comp!` | |`connectparameter` |`connect_param!` | |`setleftoverparameters` |`set_leftover_params!` | -|`setparameter` |`set_param!` | +|`setparameter` |`set_param!` | |`adddimension` |`add_dimension!` | |`setindex` |`set_dimension!` | @@ -96,9 +96,11 @@ The full API: To update an external parameter, use the functions `update_param!` and `udpate_params!` (previously known as `update_external_parameter` and `update_external_parameters`, respectively.) Their calling signatures are: -* `update_params!(md::ModelDef, parameters::Dict)` +* `update_params!(md::ModelDef, parameters::Dict; update_timesteps = false)` -* `update_param!(md::ModelDef, name::Symbol, value)` +* `update_param!(md::ModelDef, name::Symbol, value; update_timesteps = false)` + +For external parameters with a `:time` dimension, passing `update_timesteps=true` indicates that the time _keys_ (i.e., year labels) should also be updated in addition to updating the parameter values. #### Setting parameters with a dictionary diff --git a/docs/src/tutorials/tutorial_3.md b/docs/src/tutorials/tutorial_3.md index fd5ee12ea..e08567559 100644 --- a/docs/src/tutorials/tutorial_3.md +++ b/docs/src/tutorials/tutorial_3.md @@ -28,7 +28,7 @@ When the original model calls [`set_param!`](@ref), Mimi creates an external par update_param!(mymodel, :parametername, newvalues) ``` -In the code above, `newvalues` must be the same size and type (or be able to convert to the type) as required by the model dimensions set with `set_dimension!`. +Note here that `newvalues` must be the same type (or be able to convert to the type) of the old values stored in that parameter, and the same size as the model dimensions indicate. Also note that it if you have updated the time dimension of the model with `set_dimension!(m, :time, values)` you will need to update all parameters with a `:time` dimension, **even if the values have not changed**, so that the model can update the underlying time labels (ie. year labels) to match your new model time labels (ie. year labels). If you wish to alter connections within an existing model, [`disconnect_param!`](@ref) and [`connect_param!`](@ref) can be used in conjunction with each other to update the connections within the model, although this is more likely to be done as part of larger changes involving components themslves, as discussed in the next subsection. @@ -94,6 +94,8 @@ nyears = length(years) set_dimension!(m, :time, years) ``` +Now you must update at least all parameters with a `:time` dimension, even if the length and values remain the same, so that the underlying time labels (ie. year labels) update to match your new model time labels (ie. year labels). + Create a dictionary `params` with one entry `(k, v)` per external parameter by name `k` to value `v`. Each key `k` must be a symbol or convert to a symbol matching the name of an external parameter that already exists in the model definition. Part of this dictionary may look like: ```julia diff --git a/src/core/connections.jl b/src/core/connections.jl index d1155883e..516f48f20 100644 --- a/src/core/connections.jl +++ b/src/core/connections.jl @@ -479,6 +479,12 @@ function update_param!(obj::AbstractCompositeComponentDef, name::Symbol, value) _update_param!(obj::AbstractCompositeComponentDef, name, value) end +# function update_param!(obj::AbstractCompositeComponentDef, name::Symbol, value; update_timesteps = false) + +# @warn("Use of the `update_timesteps` keyword argument is no longer supported or needed, time labels will be adjusted automatically if necessary.") +# _update_param!(obj::AbstractCompositeComponentDef, name, value) +# end + function update_param!(mi::ModelInstance, name::Symbol, value) param = mi.md.external_params[name] From f00df10fa87c7f8ad0213dccd229703bbe3de400 Mon Sep 17 00:00:00 2001 From: lrennels Date: Thu, 28 Jan 2021 16:03:20 -0800 Subject: [PATCH 06/32] Add warning for update_timesteps keyword arg --- src/core/connections.jl | 29 +++++++++++++---------------- src/core/defs.jl | 11 ++++++++--- src/core/model.jl | 21 +++++++++++++++------ wip/temp.jl | 16 +++++++++++----- 4 files changed, 47 insertions(+), 30 deletions(-) diff --git a/src/core/connections.jl b/src/core/connections.jl index 516f48f20..7755b5756 100644 --- a/src/core/connections.jl +++ b/src/core/connections.jl @@ -69,14 +69,15 @@ function _check_labels(obj::AbstractCompositeComponentDef, for (i, dim) in enumerate(comp_dims) if isa(dim, Symbol) param_length = size(ext_param.values)[i] - if dim == :time - t = dimension(obj, :time) - first = find_first_period(comp_def) - last = find_last_period(comp_def) - comp_length = t[last] - t[first] + 1 - else + # we are now requiring time lengths to match the model as well + # if dim == :time + # t = dimension(obj, :time) + # first = find_first_period(comp_def) + # last = find_last_period(comp_def) + # comp_length = t[last] - t[first] + 1 + # else comp_length = dim_count(obj, dim) - end + # end if param_length != comp_length error("Mismatched data size for a parameter connection: dimension :$dim in $(comp_def.comp_id) has $comp_length elements; external parameter :$param_name has $param_length elements.") end @@ -470,21 +471,17 @@ function set_external_scalar_param!(obj::ModelDef, name::Symbol, value::Any) end """ - update_param!(obj::AbstractCompositeComponentDef, name::Symbol, value) + update_param!(obj::AbstractCompositeComponentDef, name::Symbol, value; update_timesteps = nothing) Update the `value` of an external model parameter in composite `obj`, referenced -by `name`. +by `name`. The update_timesteps keyword argument is deprecated, we keep it here +just to provide warnings. """ -function update_param!(obj::AbstractCompositeComponentDef, name::Symbol, value) +function update_param!(obj::AbstractCompositeComponentDef, name::Symbol, value; update_timesteps = nothing) + !isnothing(update_timesteps) ? @warn("Use of the `update_timesteps` keyword argument is no longer supported or needed, time labels will be adjusted automatically if necessary.") : nothing _update_param!(obj::AbstractCompositeComponentDef, name, value) end -# function update_param!(obj::AbstractCompositeComponentDef, name::Symbol, value; update_timesteps = false) - -# @warn("Use of the `update_timesteps` keyword argument is no longer supported or needed, time labels will be adjusted automatically if necessary.") -# _update_param!(obj::AbstractCompositeComponentDef, name, value) -# end - function update_param!(mi::ModelInstance, name::Symbol, value) param = mi.md.external_params[name] diff --git a/src/core/defs.jl b/src/core/defs.jl index 35cc1b880..d9033e85a 100644 --- a/src/core/defs.jl +++ b/src/core/defs.jl @@ -529,8 +529,14 @@ function set_param!(md::ModelDef, param_name::Symbol, value; dims=nothing, ignor if num_dims == 0 values = value else - # Use the first from the comp_def if it has it, else use the tree root (usu. a ModelDef) - first = first_period(md, comp_def) + # # Use the first from the comp_def if it has it, else use the tree root (usu. a ModelDef) + # first = first_period(md, comp_def) + + # Use the first from the Model def, not the component, since we now say that the + # data needs to match the dimensions of the model itself, so we need to allocate + # the full time length even if we pad it with missings. + first = first_period(md) + first === nothing && @warn "set_param!: first === nothing" if isuniform(md) @@ -538,7 +544,6 @@ function set_param!(md::ModelDef, param_name::Symbol, value; dims=nothing, ignor values = TimestepArray{FixedTimestep{first, stepsize}, T, num_dims, ti}(value) else times = time_labels(md) - # use the first from the comp_def first_index = findfirst(isequal(first), times) values = TimestepArray{VariableTimestep{(times[first_index:end]...,)}, T, num_dims, ti}(value) end diff --git a/src/core/model.jl b/src/core/model.jl index 5e3141da5..f39ace1f7 100644 --- a/src/core/model.jl +++ b/src/core/model.jl @@ -99,27 +99,32 @@ function set_leftover_params!(m::Model, parameters::Dict{T, Any}) where T end """ - update_param!(m::Model, name::Symbol, value) + update_param!(m::Model, name::Symbol, value; update_timesteps = nothing) Update the `value` of an external model parameter in model `m`, referenced by -`name`. +`name`. The update_timesteps keyword argument is deprecated, we keep it here +just to provide warnings. """ -@delegate update_param!(m::Model, name::Symbol, value) => md +@delegate update_param!(m::Model, name::Symbol, value; update_timesteps = nothing) => md """ - update_params!(m::Model, parameters::Dict{T, Any}) where T + update_params!(m::Model, parameters::Dict{T, Any}; update_timesteps = nothing) where T For each (k, v) in the provided `parameters` dictionary, `update_param!`` is called to update the external parameter by name k to value v. Each key k must be a symbol or convert to a symbol matching the name of an external parameter t -hat already exists in the -model definition. +hat already exists in the model definition. The update_timesteps keyword argument +is deprecated, we keep it here just to provide warnings. """ @delegate update_params!(m::Model, parameters::Dict) => md """ add_comp!( m::Model, comp_id::ComponentId, comp_name::Symbol=comp_id.comp_name; + first::NothingInt=nothing, + last::NothingInt=nothing, + first_free::Bool=true, + last_free::Bool=true, before::NothingSymbol=nothing, after::NothingSymbol=nothing, rename::NothingPairList=nothing @@ -138,6 +143,10 @@ end """ add_comp!( m::Model, comp_def::AbstractComponentDef, comp_name::Symbol=comp_id.comp_name; + first::NothingInt=nothing, + last::NothingInt=nothing, + first_free::Bool=true, + last_free::Bool=true, before::NothingSymbol=nothing, after::NothingSymbol=nothing, rename::NothingPairList=nothing diff --git a/wip/temp.jl b/wip/temp.jl index 236491eea..6a7d1f0ee 100644 --- a/wip/temp.jl +++ b/wip/temp.jl @@ -44,7 +44,7 @@ function construct_model() # Order matters here. If the emissions component were defined first, the model would not run. add_comp!(m, grosseconomy) - add_comp!(m, emissions, first = 2020) + add_comp!(m, emissions, first = 2020, last = 2105) # Set parameters for the grosseconomy component set_param!(m, :grosseconomy, :l, [(1. + 0.015)^t *6404 for t in 1:20]) @@ -55,7 +55,8 @@ function construct_model() set_param!(m, :grosseconomy, :share, 0.3) # Set parameters for the emissions component - set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:19]) + # set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:19]) # this should error! + set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) # Note that connect_param! was used here. @@ -65,12 +66,17 @@ end #end function m = construct_model() run(m) + +update_param!(m, :l, [(1. + 0.015)^t *6404 for t in 1:20], update_timesteps = true) # check the warning +update_param!(m, :l, [(1. + 0.015)^t *6404 for t in 1:20]) + set_dimension!(m, :time, collect(2015:5:2115)) # Set parameters for the grosseconomy component -update_param!(m, :l, [(1. + 0.015)^t *6404 for t in 1:20]) -update_param!(m, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20]) +update_param!(m, :l, [(1. + 0.015)^t *6404 for t in 1:21]) +update_param!(m, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:21]) +update_param!(m, :s, ones(21).* 0.22) # Set parameters for the emissions component -update_param!(m, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) +update_param!(m, :sigma, [(1. - 0.05)^t *0.58 for t in 1:21]) run(m) \ No newline at end of file From f840656125996998bf1a550df4bc2bccd82816bb Mon Sep 17 00:00:00 2001 From: lrennels Date: Thu, 28 Jan 2021 16:52:15 -0800 Subject: [PATCH 07/32] Add testing --- test/runtests.jl | 3 + test/test_firstlast.jl | 199 +++++++++++++++++++++++++++++++++++++++++ wip/temp.jl | 82 ----------------- 3 files changed, 202 insertions(+), 82 deletions(-) create mode 100644 test/test_firstlast.jl delete mode 100644 wip/temp.jl diff --git a/test/runtests.jl b/test/runtests.jl index ac2c4e1d4..f2fcfdb80 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -107,6 +107,9 @@ Electron.prep_test_env() @info("test_connectorcomp.jl") @time include("test_connectorcomp.jl") + @info("test_firstlast.jl") + @time include("test_firstlast.jl") + @info("test_explorer_model.jl") @time include("test_explorer_model.jl") diff --git a/test/test_firstlast.jl b/test/test_firstlast.jl new file mode 100644 index 000000000..a56645410 --- /dev/null +++ b/test/test_firstlast.jl @@ -0,0 +1,199 @@ +module TestFirstLast + +using Mimi +using Test + +import Mimi: time_labels + +# +# Define some Components +# + +@defcomp grosseconomy begin + YGROSS = Variable(index=[time]) # Gross output + K = Variable(index=[time]) # Capital + l = Parameter(index=[time]) # Labor + tfp = Parameter(index=[time]) # Total factor productivity + s = Parameter(index=[time]) # Savings rate + depk = Parameter() # Depreciation rate on capital - Note that it has no time index + k0 = Parameter() # Initial level of capital + share = Parameter() # Capital share + + function run_timestep(p, v, d, t) + if is_first(t) + v.K[t] = p.k0 + else + v.K[t] = (1 - p.depk)^5 * v.K[t-1] + v.YGROSS[t-1] * p.s[t-1] * 5 + end + v.YGROSS[t] = p.tfp[t] * v.K[t]^p.share * p.l[t]^(1-p.share) + end +end + +@defcomp grosseconomy2 begin + YGROSS = Variable(index=[time]) # Gross output + K = Variable(index=[time]) # Capital + l = Parameter(index=[time]) # Labor + tfp = Parameter(index=[time]) # Total factor productivity + s = Parameter(index=[time]) # Savings rate + depk = Parameter() # Depreciation rate on capital - Note that it has no time index + k0 = Parameter() # Initial level of capital + share = Parameter() # Capital share + + function run_timestep(p, v, d, t) + if t == TimestepValue(2020) + v.K[t] = p.k0 + else + v.K[t] = (1 - p.depk)^5 * v.K[t-1] + v.YGROSS[t-1] * p.s[t-1] * 5 + end + v.YGROSS[t] = p.tfp[t] * v.K[t]^p.share * p.l[t]^(1-p.share) + end +end + +@defcomp emissions begin + E = Variable(index=[time]) # Total greenhouse gas emissions + sigma = Parameter(index=[time]) # Emissions output ratio + YGROSS = Parameter(index=[time]) # Gross output - Note that YGROSS is now a parameter + + function run_timestep(p, v, d, t) + v.E[t] = p.YGROSS[t] * p.sigma[t] # Note the p. in front of YGROSS + end +end + +# +# Test first and last for one component +# + +m = Model() +set_dimension!(m, :time, collect(2015:5:2110)) # 20 timesteps +add_comp!(m, grosseconomy) +add_comp!(m, emissions, first = 2020, last = 2105) + +# check the attributes +@test collect(2015:5:2110) == time_labels(m.md) == [keys(m.md.namespace[:emissions].dim_dict[:time])...] == [keys(m.md.namespace[:grosseconomy].dim_dict[:time])...] +@test m.md.first == m.md.namespace[:grosseconomy].first +@test m.md.last == m.md.namespace[:grosseconomy].last +@test m.md.namespace[:emissions].first == 2020 +@test m.md.namespace[:emissions].last == 2105 + +# Set parameters for the grosseconomy component +set_param!(m, :grosseconomy, :l, [(1. + 0.015)^t *6404 for t in 1:20]) +set_param!(m, :grosseconomy, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20]) +set_param!(m, :grosseconomy, :s, ones(20).* 0.22) +set_param!(m, :grosseconomy, :depk, 0.1) +set_param!(m, :grosseconomy, :k0, 130.) +set_param!(m, :grosseconomy, :share, 0.3) + +# Set parameters for the emissions component +@test_throws ErrorException set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:19]) # needs to be length of model +set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) +connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) + +run(m) + +# test that there are missing values in :emissions but not :grosseconomy +@test ismissing(m[:emissions, :E][1]) +@test ismissing(m[:emissions, :E][20]) +@test sum(ismissing.(m[:emissions, :E][2:19])) == 0 +@test sum(ismissing.(m[:grosseconomy, :l])) == 0 + +# change the model dimension +set_dimension!(m, :time, collect(2015:5:2115)) + +# check that the first, last, and time have been updated properly +@test collect(2015:5:2115) == time_labels(m.md) == [keys(m.md.namespace[:emissions].dim_dict[:time])...] == [keys(m.md.namespace[:grosseconomy].dim_dict[:time])...] +@test m.md.first == m.md.namespace[:grosseconomy].first # grosseconomy first and last vary with model limits +@test m.md.last == m.md.namespace[:grosseconomy].last # grosseconomy first and last vary with model limits +@test m.md.namespace[:emissions].first == 2020 # emissions first and last are fixed +@test m.md.namespace[:emissions].last == 2105 # emissions first and last are fixed + +# reset parameters with a time dimension +update_param!(m, :l, [(1. + 0.015)^t *6404 for t in 1:21]) +update_param!(m, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:21]) +update_param!(m, :s, ones(21).* 0.22) +update_param!(m, :sigma, [(1. - 0.05)^t *0.58 for t in 1:21]) + +run(m) + +# test that there are missing values in :emissions but not :grosseconomy +@test ismissing(m[:emissions, :E][1]) +@test sum(ismissing.(m[:emissions, :E][20:21])) == 2 +@test sum(ismissing.(m[:emissions, :E][2:19])) == 0 +@test sum(ismissing.(m[:grosseconomy, :l])) == 0 + +# +# Test bounds - both start late - with is_first() +# + +m = Model() +set_dimension!(m, :time, collect(2015:5:2110)) # 20 timesteps +add_comp!(m, grosseconomy, first = 2020) +add_comp!(m, emissions, first = 2020) + +# Set parameters for the grosseconomy component +set_param!(m, :grosseconomy, :l, [(1. + 0.015)^t *6404 for t in 1:20]) +set_param!(m, :grosseconomy, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20]) +set_param!(m, :grosseconomy, :s, ones(20).* 0.22) +set_param!(m, :grosseconomy, :depk, 0.1) +set_param!(m, :grosseconomy, :k0, 130.) +set_param!(m, :grosseconomy, :share, 0.3) + +# Set parameters for the emissions component +set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) +connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) + +# this isn't going to run, because it doesnt set the initial value properly when it uses is_first() +@test_throws MissingException run(m) + +# +# Test bounds - both start late - without is_first() +# + +m = Model() +set_dimension!(m, :time, collect(2015:5:2110)) # 20 timesteps +add_comp!(m, grosseconomy2, first = 2020) +add_comp!(m, emissions, first = 2020) + +# Set parameters for the grosseconomy component +set_param!(m, :grosseconomy2, :l, [(1. + 0.015)^t *6404 for t in 1:20]) +set_param!(m, :grosseconomy2, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20]) +set_param!(m, :grosseconomy2, :s, ones(20).* 0.22) +set_param!(m, :grosseconomy2, :depk, 0.1) +set_param!(m, :grosseconomy2, :k0, 130.) +set_param!(m, :grosseconomy2, :share, 0.3) + +# Set parameters for the emissions component +set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) +connect_param!(m, :emissions, :YGROSS, :grosseconomy2, :YGROSS) + +# this will run! +run(m) +@test ismissing(m[:emissions, :E][20]) +@test ismissing(m[:grosseconomy, :YGROSS][20]) + + +# Test bounds - both end early +# + +m = Model() +set_dimension!(m, :time, collect(2015:5:2110)) # 20 timesteps +add_comp!(m, grosseconomy, last = 2105) +add_comp!(m, emissions, last = 2105) + +# Set parameters for the grosseconomy component +set_param!(m, :grosseconomy, :l, [(1. + 0.015)^t *6404 for t in 1:20]) +set_param!(m, :grosseconomy, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20]) +set_param!(m, :grosseconomy, :s, ones(20).* 0.22) +set_param!(m, :grosseconomy, :depk, 0.1) +set_param!(m, :grosseconomy, :k0, 130.) +set_param!(m, :grosseconomy, :share, 0.3) + +# Set parameters for the emissions component +set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) +connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) + +run(m) + +@test ismissing(m[:emissions, :E][20]) +@test ismissing(m[:grosseconomy, :YGROSS][20]) + +end #module \ No newline at end of file diff --git a/wip/temp.jl b/wip/temp.jl deleted file mode 100644 index 6a7d1f0ee..000000000 --- a/wip/temp.jl +++ /dev/null @@ -1,82 +0,0 @@ -using Mimi # start by importing the Mimi package to your space - -@defcomp grosseconomy begin - YGROSS = Variable(index=[time]) # Gross output - K = Variable(index=[time]) # Capital - l = Parameter(index=[time]) # Labor - tfp = Parameter(index=[time]) # Total factor productivity - s = Parameter(index=[time]) # Savings rate - depk = Parameter() # Depreciation rate on capital - Note that it has no time index - k0 = Parameter() # Initial level of capital - share = Parameter() # Capital share - - function run_timestep(p, v, d, t) - # Define an equation for K - if is_first(t) - # Note the use of v. and p. to distinguish between variables and parameters - v.K[t] = p.k0 - else - v.K[t] = (1 - p.depk)^5 * v.K[t-1] + v.YGROSS[t-1] * p.s[t-1] * 5 - end - - # Define an equation for YGROSS - v.YGROSS[t] = p.tfp[t] * v.K[t]^p.share * p.l[t]^(1-p.share) - end -end - -@defcomp emissions begin - E = Variable(index=[time]) # Total greenhouse gas emissions - sigma = Parameter(index=[time]) # Emissions output ratio - YGROSS = Parameter(index=[time]) # Gross output - Note that YGROSS is now a parameter - - function run_timestep(p, v, d, t) - - # Define an equation for E - v.E[t] = p.YGROSS[t] * p.sigma[t] # Note the p. in front of YGROSS - end -end - - -function construct_model() - m = Model() - - set_dimension!(m, :time, collect(2015:5:2110)) - - # Order matters here. If the emissions component were defined first, the model would not run. - add_comp!(m, grosseconomy) - add_comp!(m, emissions, first = 2020, last = 2105) - - # Set parameters for the grosseconomy component - set_param!(m, :grosseconomy, :l, [(1. + 0.015)^t *6404 for t in 1:20]) - set_param!(m, :grosseconomy, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20]) - set_param!(m, :grosseconomy, :s, ones(20).* 0.22) - set_param!(m, :grosseconomy, :depk, 0.1) - set_param!(m, :grosseconomy, :k0, 130.) - set_param!(m, :grosseconomy, :share, 0.3) - - # Set parameters for the emissions component - # set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:19]) # this should error! - set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) - connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) - # Note that connect_param! was used here. - - return m - -end #end function - -m = construct_model() -run(m) - -update_param!(m, :l, [(1. + 0.015)^t *6404 for t in 1:20], update_timesteps = true) # check the warning -update_param!(m, :l, [(1. + 0.015)^t *6404 for t in 1:20]) - -set_dimension!(m, :time, collect(2015:5:2115)) - -# Set parameters for the grosseconomy component -update_param!(m, :l, [(1. + 0.015)^t *6404 for t in 1:21]) -update_param!(m, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:21]) -update_param!(m, :s, ones(21).* 0.22) - -# Set parameters for the emissions component -update_param!(m, :sigma, [(1. - 0.05)^t *0.58 for t in 1:21]) -run(m) \ No newline at end of file From b8e730c9581142ebbe99d030b8ffc8b9ca1ad8e8 Mon Sep 17 00:00:00 2001 From: lrennels Date: Thu, 28 Jan 2021 17:18:30 -0800 Subject: [PATCH 08/32] Fix typo in test --- test/test_firstlast.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_firstlast.jl b/test/test_firstlast.jl index a56645410..48a4d9ab0 100644 --- a/test/test_firstlast.jl +++ b/test/test_firstlast.jl @@ -168,7 +168,7 @@ connect_param!(m, :emissions, :YGROSS, :grosseconomy2, :YGROSS) # this will run! run(m) @test ismissing(m[:emissions, :E][20]) -@test ismissing(m[:grosseconomy, :YGROSS][20]) +@test ismissing(m[:grosseconomy2, :YGROSS][20]) # Test bounds - both end early From 4a39aa93377b43438cf7fbfc946112066820f105 Mon Sep 17 00:00:00 2001 From: lrennels Date: Thu, 28 Jan 2021 19:29:47 -0800 Subject: [PATCH 09/32] Fix another test typo --- test/test_firstlast.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_firstlast.jl b/test/test_firstlast.jl index 48a4d9ab0..eb274bf09 100644 --- a/test/test_firstlast.jl +++ b/test/test_firstlast.jl @@ -167,8 +167,8 @@ connect_param!(m, :emissions, :YGROSS, :grosseconomy2, :YGROSS) # this will run! run(m) -@test ismissing(m[:emissions, :E][20]) -@test ismissing(m[:grosseconomy2, :YGROSS][20]) +@test ismissing(m[:emissions, :E][1]) +@test ismissing(m[:grosseconomy2, :YGROSS][1]) # Test bounds - both end early From 232544f7fbf3f10075972aee140dc3cfbe77c2ff Mon Sep 17 00:00:00 2001 From: lrennels Date: Sat, 30 Jan 2021 23:32:30 -0700 Subject: [PATCH 10/32] Reenable testing and add a getindex method --- src/components/connector.jl | 24 ++++++------ src/core/time_arrays.jl | 5 +++ test/runtests.jl | 4 +- test/test_connectorcomp.jl | 73 ++++++++++++++++--------------------- test/test_firstlast.jl | 10 +++++ test/test_parametertypes.jl | 6 +-- test/test_timesteps.jl | 14 +++---- 7 files changed, 66 insertions(+), 70 deletions(-) diff --git a/src/components/connector.jl b/src/components/connector.jl index 10f3435b9..33cb78870 100644 --- a/src/components/connector.jl +++ b/src/components/connector.jl @@ -8,14 +8,14 @@ using Mimi first = Parameter() # first year to use the shorter data last = Parameter() # last year to use the shorter data - function run_timestep(p, v, d, ts) - if gettime(ts) >= p.first && gettime(ts) <= p.last + function run_timestep(p, v, d, t) + if gettime(t) >= p.first && gettime(t) <= p.last input = p.input1 else input = p.input2 end - v.output[ts] = @allow_missing(input[ts]) + v.output[t] = @allow_missing(input[t]) end end @@ -30,16 +30,16 @@ end first = Parameter() # first year to use the shorter data last = Parameter() # last year to use the shorter data - function run_timestep(p, v, d, ts) + function run_timestep(p, v, d, t) - if gettime(ts) >= p.first && gettime(ts) <= p.last + if gettime(t) >= p.first && gettime(t) <= p.last input = p.input1 else input = p.input2 end for r in d.regions - v.output[ts, r] = @allow_missing(input[ts, r]) + v.output[t, r] = @allow_missing(input[t, r]) end end end @@ -55,14 +55,14 @@ end # output = Variable(index = [time, ...]) # # Allow copying of vars/params with arbitrary dimensions -# function run_timestep(p, v, d, ts) +# function run_timestep(p, v, d, t) # colons = repeat([:], inner=ndims(v.output) - 1) -# if hasvalue(p.input1, ts) -# v.output[ts, colons...] = p.input1[ts, colons...] -# elseif hasvalue(p.input2, ts) -# v.output[ts, colons...] = p.input2[ts, colons...] +# if hasvalue(p.input1, t) +# v.output[t, colons...] = p.input1[t, colons...] +# elseif hasvalue(p.input2, t) +# v.output[t, colons...] = p.input2[t, colons...] # else -# error("Neither of the inputs to ConnectorComp have data for the current timestep: $(gettime(ts)).") +# error("Neither of the input to ConnectorComp have data for the current timestep: $(gettime(t)).") # end # end # end diff --git a/src/core/time_arrays.jl b/src/core/time_arrays.jl index 1a5417b7a..bc6fb78fc 100644 --- a/src/core/time_arrays.jl +++ b/src/core/time_arrays.jl @@ -497,6 +497,11 @@ function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T, N, ti}, return arr.data[idxs1..., ts.t, idxs2...] end +function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP, LAST}, T, N, ti}, idxs::Union{FixedTimestep{FIRST, STEP, LAST}, AnyIndex}...) where {T, N, ti, FIRST, STEP, LAST} + idxs1, ts, idxs2 = split_indices(idxs, ti) + return arr.data[idxs1..., ts.t, idxs2...] +end + function Base.getindex(arr::TimestepArray{VariableTimestep{TIMES}, T, N, ti}, idxs::Union{VariableTimestep{TIMES}, AnyIndex}...) where {T, N, ti, TIMES} idxs1, ts, idxs2 = split_indices(idxs, ti) return arr.data[idxs1..., ts.t, idxs2...] diff --git a/test/runtests.jl b/test/runtests.jl index f2fcfdb80..dc24e7060 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -53,8 +53,8 @@ Electron.prep_test_env() @info("test_replace_comp.jl") @time include("test_replace_comp.jl") - @info("test_tools.jl") - @time include("test_tools.jl") + # @info("test_tools.jl") + # @time include("test_tools.jl") @info("test_parameter_labels.jl") @time include("test_parameter_labels.jl") diff --git a/test/test_connectorcomp.jl b/test/test_connectorcomp.jl index fee1362b9..1ae57b1a6 100644 --- a/test/test_connectorcomp.jl +++ b/test/test_connectorcomp.jl @@ -33,7 +33,7 @@ year_dim = Mimi.Dimension(years) model1 = Model() set_dimension!(model1, :time, years) -add_comp!(model1, Short) #; first=late_start) +add_comp!(model1, Short, first=late_start) add_comp!(model1, Long) set_param!(model1, :Short, :a, 2.) connect_param!(model1, :Long, :x, :Short, :b, zeros(length(years))) @@ -52,8 +52,7 @@ x = model1[:Long, :x] @test all(ismissing, b[1:year_dim[late_start]-1]) -#@test all(iszero, x[1:year_dim[late_start]-1]) -@test all(ismissing, x[1:year_dim[late_start]-1]) +@test all(iszero, x[1:year_dim[late_start]-1]) # Test the values are right after the late start @test b[year_dim[late_start]:end] == x[year_dim[late_start]:end] @@ -76,7 +75,7 @@ early_last = 2010 model2 = Model() set_dimension!(model2, :time, years_variable) -add_comp!(model2, Short) #; last=early_last) +add_comp!(model2, Short; last=early_last) add_comp!(model2, Long) set_param!(model2, :Short, :a, 2.) connect_param!(model2, :Long, :x, :Short, :b, zeros(length(years_variable))) @@ -93,16 +92,13 @@ x = model2[:Long, :x] @test length(b) == length(years_variable) @test length(x) == length(years_variable) -# -# These are no longer correct since add_comp! ignores first and last keywords -# -# @test all(ismissing, b[dim_variable[early_last]+1 : end]) -# @test all(iszero, x[dim_variable[early_last]+1 : end]) +@test all(ismissing, b[dim_variable[early_last]+1 : end]) +@test all(iszero, x[dim_variable[early_last]+1 : end]) -# # Test the values are right after the late start -# @test b[1 : dim_variable[early_last]] == -# x[1 : dim_variable[early_last]] == -# [2 * i for i in 1:dim_variable[early_last]] +# Test the values are right after the late start +@test b[dim_variable[late_start] : dim_variable[early_last]] == + x[dim_variable[late_start] : dim_variable[early_last]] == + [2 * i for i in dim_variable[late_start]:dim_variable[early_last]] #------------------------------------------------------------------------------ @@ -121,9 +117,9 @@ end a = Parameter(index=[regions]) b = Variable(index=[time, regions]) - function run_timestep(p, v, d, ts) + function run_timestep(p, v, d, t) for r in d.regions - v.b[ts, r] = ts.t + p.a[r] + v.b[t, r] = t.t + p.a[r] end end end @@ -133,7 +129,7 @@ regions = [:A, :B] model3 = Model() set_dimension!(model3, :time, years) set_dimension!(model3, :regions, regions) -add_comp!(model3, Short_multi) #; first=late_start) +add_comp!(model3, Short_multi, first=late_start) add_comp!(model3, Long_multi) set_param!(model3, :Short_multi, :a, [1,2]) connect_param!(model3, :Long_multi, :x, :Short_multi, :b, zeros(length(years), length(regions))) @@ -150,18 +146,15 @@ x = model3[:Long_multi, :x] @test size(b) == (length(years), length(regions)) @test size(x) == (length(years), length(regions)) -# -# No longer correct without first/last keywords -# -# @test all(ismissing, b[1:year_dim[late_start]-1, :]) -# @test all(iszero, x[1:year_dim[late_start]-1, :]) +@test all(ismissing, b[1:year_dim[late_start]-1, :]) +@test all(iszero, x[1:year_dim[late_start]-1, :]) -# # Test the values are right after the late start -# late_yr_idxs = year_dim[late_start]:year_dim[end] +# Test the values are right after the late start +late_yr_idxs = year_dim[late_start]:year_dim[end] -# @test b[late_yr_idxs, :] == x[year_dim[late_start]:end, :] +@test b[late_yr_idxs, :] == x[year_dim[late_start]:end, :] -# @test b[late_yr_idxs, :] == [[i + 1 for i in late_yr_idxs] [i + 2 for i in late_yr_idxs]] +@test b[late_yr_idxs, :] == [[i + 1 for i in late_yr_idxs] [i + 2 for i in late_yr_idxs]] #------------------------------------------------------------------------------ # 4. Test where the short component starts late and ends early @@ -172,7 +165,7 @@ first, last = 2002, 2007 model4 = Model() set_dimension!(model4, :time, years) set_dimension!(model4, :regions, regions) -add_comp!(model4, Short_multi) #; first=first, last=last) +add_comp!(model4, Short_multi; first=first, last=last) add_comp!(model4, Long_multi) set_param!(model4, :Short_multi, :a, [1,2]) @@ -190,18 +183,14 @@ x = model4[:Long_multi, :x] @test size(b) == (length(years), length(regions)) @test size(x) == (length(years), length(regions)) -# -# No longer correct without first/last keywords -# -# @test all(ismissing, b[1:year_dim[first]-1, :]) -# @test all(ismissing, b[year_dim[last]+1:end, :]) -# @test all(iszero, x[1:year_dim[first]-1, :]) -# @test all(iszero, x[year_dim[last]+1:end, :]) +@test all(ismissing, b[1:year_dim[first]-1, :]) +@test all(ismissing, b[year_dim[last]+1:end, :]) +@test all(iszero, x[1:year_dim[first]-1, :]) +@test all(iszero, x[year_dim[last]+1:end, :]) # Test the values are right after the late start yr_idxs = year_dim[first]:year_dim[last] @test b[yr_idxs, :] == x[yr_idxs, :] -#@test b[yr_idxs, :] == [[i + 1 for i in 1:(years[end]-late_start + 1)] [i + 2 for i in 1:(years[end]-late_start + 1)]] @test b[yr_idxs, :] == [[i + 1 for i in yr_idxs] [i + 2 for i in yr_idxs]] #------------------------------------------------------------------------------ @@ -212,17 +201,17 @@ late_start_long = 2002 model5 = Model() set_dimension!(model5, :time, years) -add_comp!(model5, Short) # ; first = late_start) -add_comp!(model5, Long) #; first = late_start_long) # starts later as well, so backup data needs to match this size +add_comp!(model5, Short; first = late_start) +add_comp!(model5, Long; first = late_start_long) # starts later as well, so backup data needs to match this size set_param!(model5, :Short, :a, 2) # A. test wrong size (needs to be length of component, not length of model) -# @test_throws ErrorException connect_param!(model5, :Long=>:x, :Short=>:b, zeros(length(years))) +@test_throws ErrorException connect_param!(model5, :Long=>:x, :Short=>:b, zeros(length(years))) @test_throws ErrorException connect_param!(model4, :Long_multi=>:x, :Short_multi=>:b, zeros(length(years), length(regions)+1)) # test case with >1 dimension # B. test no backup data provided -# @test_throws ErrorException connect_param!(model5, :Long=>:x, :Short=>:b) # Error because no backup data provided +@test_throws ErrorException connect_param!(model5, :Long=>:x, :Short=>:b) # Error because no backup data provided #------------------------------------------------------------------------------ @@ -233,15 +222,15 @@ set_param!(model5, :Short, :a, 2) @defcomp foo begin par = Parameter(index=[time]) var = Variable(index=[time]) - function run_timestep(p, v, d, ts) - v.var[ts] = p.par[ts] + function run_timestep(p, v, d, t) + v.var[t] = p.par[t] end end model6 = Model() set_dimension!(model6, :time, years) add_comp!(model6, foo, :Long; rename=[:var => :long_foo]) -add_comp!(model6, foo, :Short; rename=[:var => :short_foo]) #, first=late_start) +add_comp!(model6, foo, :Short; rename=[:var => :short_foo],first=late_start) connect_param!(model6, :Short => :par, :Long => :var) set_param!(model6, :Long, :par, years) @@ -255,7 +244,7 @@ short_var = model6[:Short, :var] @test short_par == years # The parameter has values instead of `missing` for years when this component doesn't run, # because they are coming from the longer component that did run -# @test all(ismissing, short_var[1:year_dim[late_start]-1]) +@test all(ismissing, short_var[1:year_dim[late_start]-1]) @test short_var[year_dim[late_start]:end] == years[year_dim[late_start]:end] diff --git a/test/test_firstlast.jl b/test/test_firstlast.jl index eb274bf09..b8ac04fa2 100644 --- a/test/test_firstlast.jl +++ b/test/test_firstlast.jl @@ -196,4 +196,14 @@ run(m) @test ismissing(m[:emissions, :E][20]) @test ismissing(m[:grosseconomy, :YGROSS][20]) +# Test bounds - components starting or ending before/after the model +# + +m = Model() +set_dimension!(m, :time, collect(2015:5:2110)) # 20 timesteps +@test_throws ErrorException add_comp!(m, grosseconomy, first = 2000) +add_comp!(m, grosseconomy) +@test_throws ErrorException add_comp!(m, emissions, last = 2120) + + end #module \ No newline at end of file diff --git a/test/test_parametertypes.jl b/test/test_parametertypes.jl index ea24912b6..2e6ffd7c7 100644 --- a/test/test_parametertypes.jl +++ b/test/test_parametertypes.jl @@ -133,13 +133,9 @@ end m = Model() set_dimension!(m, :time, 2000:2002) -add_comp!(m, MyComp2) # ; first=2000, last=2002) +add_comp!(m, MyComp2, first=2000, last=2002) set_param!(m, :MyComp2, :x, [1, 2, 3]) -# N.B. `first` and `last` are now disabled. -# Can't move last beyond last for a component -# @test_throws ErrorException set_dimension!(m, :time, 2001:2003) - set_dimension!(m, :time, 2000:2001) set_dimension!(m, :time, 2001:2002) diff --git a/test/test_timesteps.jl b/test/test_timesteps.jl index 0fe9437e7..9302fd554 100644 --- a/test/test_timesteps.jl +++ b/test/test_timesteps.jl @@ -138,12 +138,11 @@ first_foo = 2005 m = Model() set_dimension!(m, :time, years) -# first and last disabled # test that you can only add components with first/last within model's time index range -# @test_throws ErrorException add_comp!(m, Foo; first=1900) -# @test_throws ErrorException add_comp!(m, Foo; last=2100) +@test_throws ErrorException add_comp!(m, Foo; first=1900) +@test_throws ErrorException add_comp!(m, Foo; last=2100) -foo = add_comp!(m, Foo) # DISABLED: first=first_foo) # offset for foo +foo = add_comp!(m, Foo, first=first_foo) bar = add_comp!(m, Bar) set_param!(m, :Foo, :inputF, 5.) @@ -214,12 +213,9 @@ end m2 = Model() set_dimension!(m2, :time, years) bar = add_comp!(m2, Bar) -foo2 = add_comp!(m2, Foo2) # , first=first_foo) +foo2 = add_comp!(m2, Foo2, first = first_foo) set_param!(m2, :Bar, :inputB, collect(1:length(years))) - -# TBD: Connecting components with different "first" times creates a mismatch -# in understanding how to translate the index back to a year. connect_param!(m2, :Foo2, :inputF, :Bar, :output) run(m2) @@ -247,7 +243,7 @@ years = 2000:2010 m3 = Model() set_dimension!(m3, :time, years) -add_comp!(m3, Foo) #, first=2005) +add_comp!(m3, Foo, first=2005) add_comp!(m3, Bar2) set_param!(m3, :Foo, :inputF, 5.) From 0bf8568d61a47e7bffc9beb65b0afea821712bcb Mon Sep 17 00:00:00 2001 From: lrennels Date: Sun, 31 Jan 2021 00:49:47 -0700 Subject: [PATCH 11/32] Add more tests and a doc note --- docs/src/howto/howto_6.md | 2 ++ test/runtests.jl | 12 +++++++----- test/test_components.jl | 32 +++++++++++--------------------- test/test_dimensions.jl | 19 ++++--------------- 4 files changed, 24 insertions(+), 41 deletions(-) diff --git a/docs/src/howto/howto_6.md b/docs/src/howto/howto_6.md index 8560e3f0e..12ef7db53 100644 --- a/docs/src/howto/howto_6.md +++ b/docs/src/howto/howto_6.md @@ -162,6 +162,8 @@ replace!(m, old => new) *The Mimi Change:* +**Update: This Functionality has been reenabled, please feel free to use it again, your old code should now be valid again.** + Through Mimi v0.9.4, the optional keyword arguments `first` and `last` could be used to specify times for components that do not run for the full length of the model, like this: `add_comp!(mymodel, ComponentC; first=2010, last=2100)`. This functionality is still disabled, as it was starting in v0.9.5, and all components must run for the full length of the model's time dimension. This functionality may be re-implemented in a later version of Mimi. *The User Change:* diff --git a/test/runtests.jl b/test/runtests.jl index dc24e7060..a79c1cb16 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,7 +13,7 @@ using Documenter Electron.prep_test_env() @testset "Mimi" begin - + @info("test_main.jl") @time include("test_main.jl") @@ -53,8 +53,8 @@ Electron.prep_test_env() @info("test_replace_comp.jl") @time include("test_replace_comp.jl") - # @info("test_tools.jl") - # @time include("test_tools.jl") + @info("test_tools.jl") + @time include("test_tools.jl") @info("test_parameter_labels.jl") @time include("test_parameter_labels.jl") @@ -106,10 +106,12 @@ Electron.prep_test_env() @info("test_connectorcomp.jl") @time include("test_connectorcomp.jl") - + + # (temporarily) verbose first and last keyword arguments tests while + # reenabling this functionality @info("test_firstlast.jl") @time include("test_firstlast.jl") - + @info("test_explorer_model.jl") @time include("test_explorer_model.jl") diff --git a/test/test_components.jl b/test/test_components.jl index 30260bfee..a654b951a 100644 --- a/test/test_components.jl +++ b/test/test_components.jl @@ -112,10 +112,8 @@ m = Model() set_dimension!(m, :time, 2001:2005) add_comp!(m, testcomp1, :C) # Don't set the first and last values here cd = compdef(m.md, :C) # Get the component definition in the model - -# These tests are not valid in the composite world... -#@test cd.first === nothing # First and last values should still be nothing because they were not explicitly set -#@test cd.last === nothing +@test cd.first === 2001 +@test cd.last === 2005 set_param!(m, :C, :par1, zeros(5)) Mimi.build!(m) # Build the model @@ -125,10 +123,8 @@ ci = compinstance(m, :C) # Get the component instance set_dimension!(m, :time, 2005:2020) # Reset the time dimension cd = compdef(m.md, :C) # Get the component definition in the model - -# These tests are not valid in the composite world... -#@test cd.first === nothing # First and last values should still be nothing -#@test cd.last === nothing +@test cd.first === 2005 +@test cd.last === 2020 update_param!(m, :par1, zeros(16)) Mimi.build!(m) # Build the model @@ -142,17 +138,11 @@ ci = compinstance(m, :C) # Get the component instance m = Model() set_dimension!(m, :time, 2000:2100) -add_comp!(m, testcomp1, :C) +add_comp!(m, testcomp1, :C; first = 2010, last = 2090) cd = compdef(m.md, :C) # Get the component definition in the model - -# first and last are disabled currently -# @test cd.first == 2010 # First and last values are defined in the comp def because they were explicitly given -# @test cd.last == 2090 - -# Verify that they didn't change -#@test cd.first === nothing -#@test cd.last === nothing +@test cd.first == 2010 # First and last values are defined in the comp def because they were explicitly given +@test cd.last == 2090 set_dimension!(m, :time, 2010:2090) @@ -164,12 +154,12 @@ ci = compinstance(m, :C) # Get the component instance set_dimension!(m, :time, 2000:2200) # Reset the time dimension cd = compdef(m.md, :C) # Get the component definition in the model -# @test cd.first == 2010 # First and last values should still be the same -# @test cd.last == 2090 +@test cd.first == 2010 # First and last values should still be the same +@test cd.last == 2090 Mimi.build!(m) # Build the model ci = compinstance(m, :C) # Get the component instance -# @test ci.first == 2010 # The component instance's first and last values are the same as the comp def -# @test ci.last == 2090 +@test ci.first == 2010 # The component instance's first and last values are the same as the comp def +@test ci.last == 2090 end #module diff --git a/test/test_dimensions.jl b/test/test_dimensions.jl index a4fa0dcaa..b98f97d48 100644 --- a/test/test_dimensions.jl +++ b/test/test_dimensions.jl @@ -13,9 +13,6 @@ dim_range = Dimension(2010:2100) # AbstractRange rangedim = RangeDimension(2010:2100) # RangeDimension type dim_vals = Dimension(4) # Same as 1:4 -# RJP: unclear what was intended here, but you cannot instantiate an abstract type... -# dim_vals_abstract = AbstractDimension(dim_vals) # Abstract - @test key_type(dim_varargs) == Symbol @test key_type(dim_vec) == Symbol @test key_type(dim_range) == Int @@ -47,8 +44,6 @@ end @test dim_varargs[:bar] == 2 @test dim_varargs[:] == [1,2,3] -# @test dim_vals_abstract[1:4...]== [1:4...] - # @test rangedim[2011] == 2 # TODO: this errors.. @test get(dim_varargs, :bar, 999) == 2 @@ -92,23 +87,17 @@ end m = Model() set_dimension!(m, :time, 2000:2100) -# First and last have been disabled... -#@test_throws ErrorException add_comp!(m, foo2; first = 2005, last = 2105) # Can't add a component longer than a model +@test_throws ErrorException add_comp!(m, foo2; first = 2005, last = 2105) # Can't add a component longer than a model foo2_ref = add_comp!(m, foo2) foo2_ref = ComponentReference(m, :foo2) my_foo2 = compdef(foo2_ref) -# First and last have been disabled... -# Can't set time more narrowly than components are defined as -# @test_throws ErrorException set_dimension!(m, :time, 1990:2200) -# @test first_period(my_foo2) == 2005 -# @test last_period(my_foo2) == 2095 - # Test parameter connections -# @test_throws ErrorException set_param!(m, :foo2, :x, 1990:2200) # too long -# set_param!(m, :foo2, :x, 2005:2095) # Shouldn't throw an error +@test_throws ErrorException set_param!(m, :foo2, :x, 1990:2200) # too long +@test_throws ErrorException set_param!(m, :foo2, :x, 2005:2095) # too short +set_param!(m, :foo2, :x, 2000:2100) #Shouldn't error set_dimension!(m, :time, 2010:2050) From a6a530e0a47dc8e383de3a4ea7b802b715f60e31 Mon Sep 17 00:00:00 2001 From: lrennels Date: Sun, 31 Jan 2021 11:34:15 -0700 Subject: [PATCH 12/32] Edit some documentation --- docs/src/howto/howto_1.md | 6 ++++++ docs/src/tutorials/tutorial_4.md | 2 +- src/core/defs.jl | 10 ++++++++-- src/core/model.jl | 10 ++++++++-- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/docs/src/howto/howto_1.md b/docs/src/howto/howto_1.md index 50e1cc1af..21797c166 100644 --- a/docs/src/howto/howto_1.md +++ b/docs/src/howto/howto_1.md @@ -73,6 +73,12 @@ add_comp!(m, ComponentA, :GDP) The first argument to `add_comp!` is the model, the second is the name of the ComponentId defined by `@defcomp`. If an optional third symbol is provided (as in the second line above), this will be used as the name of the component in this model. This allows you to add multiple versions of the same component to a model, with different names. +The `add_comp` function has two more optional keyword arguments, `first` and `last`, which can be used to indicate a fixed start and/or end time (year in this case) that the compnonent should run for (within the bounds of the model's time dimension). For example, the following sould indicate that `ComponentA` should only run from 1900 to 2000. + +```julia +add_comp!(m, ComponentA; first = 1900, last = 2000) +``` + The next step is to set the values for all the parameters in the components. Parameters can either have their values assigned from external data, or they can internally connect to the values from variables in other components of the model. To make an external connection, the syntax is as follows: diff --git a/docs/src/tutorials/tutorial_4.md b/docs/src/tutorials/tutorial_4.md index 5174edbfb..405c02efc 100644 --- a/docs/src/tutorials/tutorial_4.md +++ b/docs/src/tutorials/tutorial_4.md @@ -81,7 +81,7 @@ end We can now use Mimi to construct a model that binds the `grosseconomy` and `emissions` components together in order to solve for the emissions level of the global economy over time. In this example, we will run the model for twenty periods with a timestep of five years between each period. * Once the model is defined, [`set_dimension!`](@ref) is used to set the length and interval of the time step. -* We then use [`add_comp!`](@ref) to incorporate each component that we previously created into the model. It is important to note that the order in which the components are listed here matters. The model will run through each equation of the first component before moving onto the second component. +* We then use [`add_comp!`](@ref) to incorporate each component that we previously created into the model. It is important to note that the order in which the components are listed here matters. The model will run through each equation of the first component before moving onto the second component. One can also use the optional `first` and `last` keyword arguments to indicate a subset of the model's time dimension when the component should start and end. * Next, [`set_param!`](@ref) is used to assign values to each parameter in the model, with parameters being uniquely tied to each component. If _population_ was a parameter for two different components, it must be assigned to each one using [`set_param!`](@ref) two different times. The syntax is `set_param!(model_name, :component_name, :parameter_name, value)` * If any variables of one component are parameters for another, [`connect_param!`](@ref) is used to couple the two components together. In this example, _YGROSS_ is a variable in the `grosseconomy` component and a parameter in the `emissions` component. The syntax is `connect_param!(model_name, :component_name_parameter, :parameter_name, :component_name_variable, :variable_name)`, where `:component_name_variable` refers to the component where your parameter was initially calculated as a variable. * Finally, the model can be run using the command `run(model_name)`. diff --git a/src/core/defs.jl b/src/core/defs.jl index d9033e85a..20ef69304 100644 --- a/src/core/defs.jl +++ b/src/core/defs.jl @@ -826,7 +826,10 @@ end Add the component `comp_def` to the composite component indicated by `obj`. The component is added at the end of the list unless one of the keywords `before` or `after` is specified. Note that a copy of `comp_id` is made in the composite and assigned the give name. The optional -argument `rename` can be a list of pairs indicating `original_name => imported_name`. +argument `rename` can be a list of pairs indicating `original_name => imported_name`. The optional +arguments `first` and `last` indicate the times bounding the run period for the given component, +which must be within the bounds of the model and if explicitly set are fixed. These default +to flexibly changing with the model's `:time` dimension. """ function add_comp!(obj::AbstractCompositeComponentDef, comp_def::AbstractComponentDef, @@ -883,7 +886,10 @@ end Add the component indicated by `comp_id` to the composite component indicated by `obj`. The component is added at the end of the list unless one of the keywords `before` or `after` is -specified. Note that a copy of `comp_id` is made in the composite and assigned the give name. +specified. Note that a copy of `comp_id` is made in the composite and assigned the give name. The optional +arguments `first` and `last` indicate the times bounding the run period for the given component, +which must be within the bounds of the model and if explicitly set are fixed. These default +to flexibly changing with the model's `:time` dimension. [Not yet implemented:] The optional argument `rename` can be a list of pairs indicating `original_name => imported_name`. diff --git a/src/core/model.jl b/src/core/model.jl index f39ace1f7..ca14dbb81 100644 --- a/src/core/model.jl +++ b/src/core/model.jl @@ -133,7 +133,10 @@ is deprecated, we keep it here just to provide warnings. Add the component indicated by `comp_id` to the model indicated by `m`. The component is added at the end of the list unless one of the keywords `before` or `after` is specified. Note that a copy of `comp_id` is made in the composite and assigned the give name. The optional -argument `rename` can be a list of pairs indicating `original_name => imported_name`. +argument `rename` can be a list of pairs indicating `original_name => imported_name`. The optional +arguments `first` and `last` indicate the times bounding the run period for the given component, +which must be within the bounds of the model and if explicitly set are fixed. These default +to flexibly changing with the model's `:time` dimension. """ function add_comp!(m::Model, comp_id::ComponentId, comp_name::Symbol=comp_id.comp_name; kwargs...) comp_def = add_comp!(m.md, comp_id, comp_name; kwargs...) @@ -155,7 +158,10 @@ end Add the component `comp_def` to the model indicated by `m`. The component is added at the end of the list unless one of the keywords, `first`, `last`, `before`, `after`. Note that a copy of `comp_id` is made in the composite and assigned the give name. The optional -argument `rename` can be a list of pairs indicating `original_name => imported_name`. +argument `rename` can be a list of pairs indicating `original_name => imported_name`. The optional +arguments `first` and `last` indicate the times bounding the run period for the given component, +which must be within the bounds of the model and if explicitly set are fixed. These default +to flexibly changing with the model's `:time` dimension. """ function add_comp!(m::Model, comp_def::AbstractComponentDef, comp_name::Symbol=comp_def.comp_id.comp_name; kwargs...) return add_comp!(m, comp_def.comp_id, comp_name; kwargs...) From 2e655c585c50d7f4840d7683a16a0b8cde003f09 Mon Sep 17 00:00:00 2001 From: lrennels Date: Wed, 3 Feb 2021 15:50:13 -0700 Subject: [PATCH 13/32] Cleanup and docs --- test/test_clock.jl | 1 - test/test_firstlast.jl | 31 ++++++++++++++++++++---------- test/test_getdataframe.jl | 5 ++--- test/test_main.jl | 2 -- test/test_main_variabletimestep.jl | 2 -- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/test/test_clock.jl b/test/test_clock.jl index bfe7b2aff..535322135 100644 --- a/test/test_clock.jl +++ b/test/test_clock.jl @@ -14,7 +14,6 @@ c_f = Clock{FixedTimestep}(1850, 10, 3000) advance(c_f) @test time_index(c_f) == 2 - years = Tuple([2000:1:2024; 2025:5:2105]) t_v = VariableTimestep{years}() c_v = Clock{VariableTimestep}(years) diff --git a/test/test_firstlast.jl b/test/test_firstlast.jl index b8ac04fa2..75839ca81 100644 --- a/test/test_firstlast.jl +++ b/test/test_firstlast.jl @@ -60,7 +60,7 @@ end end # -# Test first and last for one component +# Test using first and last for one (the second) component # m = Model() @@ -68,7 +68,8 @@ set_dimension!(m, :time, collect(2015:5:2110)) # 20 timesteps add_comp!(m, grosseconomy) add_comp!(m, emissions, first = 2020, last = 2105) -# check the attributes +# check that the attributes of the ModelDef and ComponentDef(s) have been set_dimension +# as expected @test collect(2015:5:2110) == time_labels(m.md) == [keys(m.md.namespace[:emissions].dim_dict[:time])...] == [keys(m.md.namespace[:grosseconomy].dim_dict[:time])...] @test m.md.first == m.md.namespace[:grosseconomy].first @test m.md.last == m.md.namespace[:grosseconomy].last @@ -84,29 +85,31 @@ set_param!(m, :grosseconomy, :k0, 130.) set_param!(m, :grosseconomy, :share, 0.3) # Set parameters for the emissions component -@test_throws ErrorException set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:19]) # needs to be length of model +@test_throws ErrorException set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:19]) # the parameter needs to be length of model set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) run(m) -# test that there are missing values in :emissions but not :grosseconomy +# test that there are missing values in :emissions variables outside of the component's +# run period, and no missing values in the :grosseconomy variables @test ismissing(m[:emissions, :E][1]) @test ismissing(m[:emissions, :E][20]) @test sum(ismissing.(m[:emissions, :E][2:19])) == 0 @test sum(ismissing.(m[:grosseconomy, :l])) == 0 -# change the model dimension +# change the model dimension (widen it) set_dimension!(m, :time, collect(2015:5:2115)) -# check that the first, last, and time have been updated properly +# check that the first, last, and time have been updated properly for both the +# ModelDef and ComponentDef(s) @test collect(2015:5:2115) == time_labels(m.md) == [keys(m.md.namespace[:emissions].dim_dict[:time])...] == [keys(m.md.namespace[:grosseconomy].dim_dict[:time])...] @test m.md.first == m.md.namespace[:grosseconomy].first # grosseconomy first and last vary with model limits @test m.md.last == m.md.namespace[:grosseconomy].last # grosseconomy first and last vary with model limits @test m.md.namespace[:emissions].first == 2020 # emissions first and last are fixed @test m.md.namespace[:emissions].last == 2105 # emissions first and last are fixed -# reset parameters with a time dimension +# reset any parameters that have a time dimension update_param!(m, :l, [(1. + 0.015)^t *6404 for t in 1:21]) update_param!(m, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:21]) update_param!(m, :s, ones(21).* 0.22) @@ -114,7 +117,8 @@ update_param!(m, :sigma, [(1. - 0.05)^t *0.58 for t in 1:21]) run(m) -# test that there are missing values in :emissions but not :grosseconomy +# test that there are missing values in :emissions variables outside of the component's +# run period, and no missing values in the :grosseconomy variables @test ismissing(m[:emissions, :E][1]) @test sum(ismissing.(m[:emissions, :E][20:21])) == 2 @test sum(ismissing.(m[:emissions, :E][2:19])) == 0 @@ -141,7 +145,8 @@ set_param!(m, :grosseconomy, :share, 0.3) set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) -# this isn't going to run, because it doesnt set the initial value properly when it uses is_first() +# this isn't going to run, because it doesnt set the initial value properly when +# it uses is_first() ... we will test fixing this problem in the next test @test_throws MissingException run(m) # @@ -165,8 +170,11 @@ set_param!(m, :grosseconomy2, :share, 0.3) set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) connect_param!(m, :emissions, :YGROSS, :grosseconomy2, :YGROSS) -# this will run! +# this will run because we explicitly used `TimestepValue(2020)` instead of +# using is_first() run(m) + +# neither component should have a value for the first timestep @test ismissing(m[:emissions, :E][1]) @test ismissing(m[:grosseconomy2, :YGROSS][1]) @@ -196,11 +204,14 @@ run(m) @test ismissing(m[:emissions, :E][20]) @test ismissing(m[:grosseconomy, :YGROSS][20]) +# # Test bounds - components starting or ending before/after the model # m = Model() set_dimension!(m, :time, collect(2015:5:2110)) # 20 timesteps + +# components cannot start before or end after the model's time dimension @test_throws ErrorException add_comp!(m, grosseconomy, first = 2000) add_comp!(m, grosseconomy) @test_throws ErrorException add_comp!(m, emissions, last = 2120) diff --git a/test/test_getdataframe.jl b/test/test_getdataframe.jl index f0c020aed..812b42a4c 100644 --- a/test/test_getdataframe.jl +++ b/test/test_getdataframe.jl @@ -62,12 +62,11 @@ run(model1) df = getdataframe(model1, :testcomp1=>:var1, :testcomp1=>:par1, :testcomp2=>:var2, :testcomp2=>:par2) dim = Mimi.dimension(model1, :time) -@test df.var1 == df.par1 == years +@test df.var1 == df.par1 == df.par2 == years @test all(ismissing, df.var2[1 : dim[late_first]-1]) -#@test all(ismissing, df.par2[1 : dim[late_first]-1]) @test df.var2[dim[late_first] : dim[early_last]] == df.par2[dim[late_first] : dim[early_last]] == late_first:5:early_last @test all(ismissing, df.var2[dim[years[end]] : dim[early_last]]) -@test all(ismissing, df.par2[dim[years[end]] : dim[early_last]]) +@test all(ismissing, df.var2[dim[early_last]+1 : dim[years[end]]]) # Test trying to load an item into an existing dataframe where that item key already exists @test_throws UndefVarError _load_dataframe(model1, :testcomp1, :var1, df) diff --git a/test/test_main.jl b/test/test_main.jl index 8cd2c8cc9..ef79ef62d 100644 --- a/test/test_main.jl +++ b/test/test_main.jl @@ -35,8 +35,6 @@ set_param!(x1, :foo1, :par1, 5.0) @test length(dimension(x1.md, :index1)) == 3 -# @test_throws MethodError x1.Parameters.par1 = Array{Float64}(undef, 1, 2) - par1 = external_param(x1, :par1) @test par1.value == 5.0 diff --git a/test/test_main_variabletimestep.jl b/test/test_main_variabletimestep.jl index cc0ab5790..a389611f5 100644 --- a/test/test_main_variabletimestep.jl +++ b/test/test_main_variabletimestep.jl @@ -35,8 +35,6 @@ set_param!(x1, :foo1, :par1, 5.0) @test length(dimension(x1.md, :index1)) == 3 -# @test_throws MethodError x1.Parameters.par1 = Array{Float64}(1,2) - par1 = external_param(x1, :par1) @test par1.value == 5.0 From 8989198ce48569da79731c4ac21900b7b2a79b38 Mon Sep 17 00:00:00 2001 From: lrennels Date: Mon, 8 Feb 2021 22:43:57 -0700 Subject: [PATCH 14/32] Add get_shifted_ts functions --- src/core/instances.jl | 15 ++++++++++++- test/test_firstlast.jl | 51 ++---------------------------------------- 2 files changed, 16 insertions(+), 50 deletions(-) diff --git a/src/core/instances.jl b/src/core/instances.jl index 6fab75bfd..10b729705 100644 --- a/src/core/instances.jl +++ b/src/core/instances.jl @@ -278,9 +278,22 @@ end _runnable(ci::AbstractComponentInstance, clock::Clock) = (ci.first <= gettime(clock) <= ci.last) +function get_shifted_ts(ci, ts::FixedTimestep{FIRST, STEP, LAST}) where {FIRST, STEP, LAST} + shift = Int((ci.first - FIRST)/STEP) + return FixedTimestep{ci.first,STEP,ci.last}(ts.t - shift) +end + +function get_shifted_ts(ci, ts::VariableTimestep{TIMES}) where {TIMES} + idx_start = findfirst(TIMES .== ci.first) + idx_end = findfirst(TIMES .== ci.last) + shift = idx_start - 1 + + return VariableTimestep{TIMES[idx_start:idx_end]}(ts.t - shift) +end + function run_timestep(ci::AbstractComponentInstance, clock::Clock, dims::DimValueDict) if ci.run_timestep !== nothing && _runnable(ci, clock) - ci.run_timestep(parameters(ci), variables(ci), dims, clock.ts) + ci.run_timestep(parameters(ci), variables(ci), dims, get_shifted_ts(ci, clock.ts)) end return nothing diff --git a/test/test_firstlast.jl b/test/test_firstlast.jl index 75839ca81..5f69a3db7 100644 --- a/test/test_firstlast.jl +++ b/test/test_firstlast.jl @@ -29,26 +29,6 @@ import Mimi: time_labels end end -@defcomp grosseconomy2 begin - YGROSS = Variable(index=[time]) # Gross output - K = Variable(index=[time]) # Capital - l = Parameter(index=[time]) # Labor - tfp = Parameter(index=[time]) # Total factor productivity - s = Parameter(index=[time]) # Savings rate - depk = Parameter() # Depreciation rate on capital - Note that it has no time index - k0 = Parameter() # Initial level of capital - share = Parameter() # Capital share - - function run_timestep(p, v, d, t) - if t == TimestepValue(2020) - v.K[t] = p.k0 - else - v.K[t] = (1 - p.depk)^5 * v.K[t-1] + v.YGROSS[t-1] * p.s[t-1] * 5 - end - v.YGROSS[t] = p.tfp[t] * v.K[t]^p.share * p.l[t]^(1-p.share) - end -end - @defcomp emissions begin E = Variable(index=[time]) # Total greenhouse gas emissions sigma = Parameter(index=[time]) # Emissions output ratio @@ -145,38 +125,11 @@ set_param!(m, :grosseconomy, :share, 0.3) set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) -# this isn't going to run, because it doesnt set the initial value properly when -# it uses is_first() ... we will test fixing this problem in the next test -@test_throws MissingException run(m) - -# -# Test bounds - both start late - without is_first() -# - -m = Model() -set_dimension!(m, :time, collect(2015:5:2110)) # 20 timesteps -add_comp!(m, grosseconomy2, first = 2020) -add_comp!(m, emissions, first = 2020) - -# Set parameters for the grosseconomy component -set_param!(m, :grosseconomy2, :l, [(1. + 0.015)^t *6404 for t in 1:20]) -set_param!(m, :grosseconomy2, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20]) -set_param!(m, :grosseconomy2, :s, ones(20).* 0.22) -set_param!(m, :grosseconomy2, :depk, 0.1) -set_param!(m, :grosseconomy2, :k0, 130.) -set_param!(m, :grosseconomy2, :share, 0.3) - -# Set parameters for the emissions component -set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) -connect_param!(m, :emissions, :YGROSS, :grosseconomy2, :YGROSS) - -# this will run because we explicitly used `TimestepValue(2020)` instead of -# using is_first() -run(m) +run(m) # neither component should have a value for the first timestep @test ismissing(m[:emissions, :E][1]) -@test ismissing(m[:grosseconomy2, :YGROSS][1]) +@test ismissing(m[:grosseconomy, :YGROSS][1]) # Test bounds - both end early From 6939783861d425910ccd9f64db3b79e07c8c3cc2 Mon Sep 17 00:00:00 2001 From: lrennels Date: Mon, 8 Feb 2021 23:42:54 -0700 Subject: [PATCH 15/32] Fix up expected test values --- test/runtests.jl | 2 -- test/test_connectorcomp.jl | 10 +++++++--- test/test_timesteps.jl | 4 +++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 296db1949..557b89007 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -107,8 +107,6 @@ Electron.prep_test_env() @info("test_connectorcomp.jl") @time include("test_connectorcomp.jl") - # (temporarily) verbose first and last keyword arguments tests while - # reenabling this functionality @info("test_firstlast.jl") @time include("test_firstlast.jl") diff --git a/test/test_connectorcomp.jl b/test/test_connectorcomp.jl index 1ae57b1a6..5c6850a42 100644 --- a/test/test_connectorcomp.jl +++ b/test/test_connectorcomp.jl @@ -55,8 +55,9 @@ x = model1[:Long, :x] @test all(iszero, x[1:year_dim[late_start]-1]) # Test the values are right after the late start +offset = late_start - years[1] @test b[year_dim[late_start]:end] == x[year_dim[late_start]:end] -@test b[year_dim[late_start]:end] == collect(year_dim[late_start]:year_dim[years[end]]) * 2.0 +@test b[year_dim[late_start]:end] == (collect(year_dim[late_start]:year_dim[years[end]]) .- 5) * 2.0 @test Mimi.datum_size(model1.md, Mimi.compdef(model1.md, :Long), :x) == (length(years),) @@ -154,7 +155,8 @@ late_yr_idxs = year_dim[late_start]:year_dim[end] @test b[late_yr_idxs, :] == x[year_dim[late_start]:end, :] -@test b[late_yr_idxs, :] == [[i + 1 for i in late_yr_idxs] [i + 2 for i in late_yr_idxs]] +offset = late_start - years[1] +@test b[late_yr_idxs, :] == [[i + 1 - offset for i in late_yr_idxs] [i + 2 - offset for i in late_yr_idxs]] #------------------------------------------------------------------------------ # 4. Test where the short component starts late and ends early @@ -189,9 +191,11 @@ x = model4[:Long_multi, :x] @test all(iszero, x[year_dim[last]+1:end, :]) # Test the values are right after the late start + +offset = first - years[1] yr_idxs = year_dim[first]:year_dim[last] @test b[yr_idxs, :] == x[yr_idxs, :] -@test b[yr_idxs, :] == [[i + 1 for i in yr_idxs] [i + 2 for i in yr_idxs]] +@test b[yr_idxs, :] == [[i + 1 - offset for i in yr_idxs] [i + 2 - offset for i in yr_idxs]] #------------------------------------------------------------------------------ # 5. Test errors with backup data diff --git a/test/test_timesteps.jl b/test/test_timesteps.jl index 9302fd554..d406fc65d 100644 --- a/test/test_timesteps.jl +++ b/test/test_timesteps.jl @@ -156,8 +156,10 @@ run(m) yr_dim = Mimi.Dimension(years) idxs = yr_dim[first_foo]:yr_dim[years[end]] foo_output = m[:Foo, :output] + +offset = first_foo - years[1] for i in idxs - @test foo_output[i] == 5+i + @test foo_output[i] == 5+(i-offset) # incorporate offset into i now because we set ts.t to match component not model end for i in 1:5 From 1f0e5a122dc479c0c2c509c6e4f09f2178118488 Mon Sep 17 00:00:00 2001 From: lrennels Date: Tue, 9 Feb 2021 00:02:50 -0700 Subject: [PATCH 16/32] Add testing --- test/test_firstlast.jl | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/test_firstlast.jl b/test/test_firstlast.jl index 5f69a3db7..230ff91c4 100644 --- a/test/test_firstlast.jl +++ b/test/test_firstlast.jl @@ -169,5 +169,46 @@ set_dimension!(m, :time, collect(2015:5:2110)) # 20 timesteps add_comp!(m, grosseconomy) @test_throws ErrorException add_comp!(m, emissions, last = 2120) +# +# Test is_first and is_last +# + +# is_first and is_last should be equivalent to comparing the timestep to the first +# and last years provided, and t.t should be 1 when the year is first i.e. the +# first year the component actually runs for. + +@defcomp MyComp begin + a = Variable(index=[time]) + function run_timestep(p, v, d, t) + if t == TimestepValue(2) v.a[t] = -999 + elseif t == TimestepValue(9) v.a[t] = 999 + else v.a[t] = t.t + end + end +end + +@defcomp MyComp2 begin + a = Variable(index=[time]) + function run_timestep(p, v, d, t) + if is_first(t) v.a[t] = -999 + elseif is_last(t) v.a[t] = 999 + else v.a[t] = t.t + end + end +end + +m = Model() +set_dimension!(m, :time, collect(1:10)) +add_comp!(m, MyComp, first = 2, last = 9) +run(m) + +m2 = Model() +set_dimension!(m2, :time, collect(1:10)) +add_comp!(m2, MyComp, first = 2, last = 9) +run(m2) + +@test ismissing(m[:MyComp, :a][1]) +@test ismissing(m[:MyComp, :a][end]) +@test (m[:MyComp, :a])[2:9] == (m2[:MyComp, :a])[2:9] == [-999., 2., 3., 4., 5., 6., 7., 999.] end #module \ No newline at end of file From bf59ee8cd7fdc550510095f85b58cb139d2af12c Mon Sep 17 00:00:00 2001 From: lrennels Date: Tue, 9 Feb 2021 00:56:59 -0700 Subject: [PATCH 17/32] Add performance improvements --- docs/src/howto/howto_4.md | 4 ++-- src/core/instances.jl | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/docs/src/howto/howto_4.md b/docs/src/howto/howto_4.md index 8cf62b26c..d1c1052b1 100644 --- a/docs/src/howto/howto_4.md +++ b/docs/src/howto/howto_4.md @@ -7,8 +7,8 @@ An `AbstractTimestep` i.e. a `FixedTimestep` or a `VariableTimestep` is a type d In the `run_timestep` functions which the user defines, it may be useful to use any of the following functions, where `t` is an `AbstractTimestep` object: ```julia -is_first(t) # returns true or false, true if t is the first timestep to be run -is_last(t) # returns true or false, true if t is the last timestep to be run +is_first(t) # returns true or false, true if t is the first timestep to be run for the respective component +is_last(t) # returns true or false, true if t is the last timestep to be run for the respective component gettime(t) # returns the year represented by timestep t ``` There are also two helper types `TimestepValue` and `TimestepIndex` that can be used with comparison operators (`==`, `<`, and `>`) to check whether an `AbstractTimestep` `t` during the `run_timestep` function corresponds with a certain year or index number. For example: diff --git a/src/core/instances.jl b/src/core/instances.jl index 10b729705..36e224f5d 100644 --- a/src/core/instances.jl +++ b/src/core/instances.jl @@ -279,16 +279,23 @@ end _runnable(ci::AbstractComponentInstance, clock::Clock) = (ci.first <= gettime(clock) <= ci.last) function get_shifted_ts(ci, ts::FixedTimestep{FIRST, STEP, LAST}) where {FIRST, STEP, LAST} - shift = Int((ci.first - FIRST)/STEP) - return FixedTimestep{ci.first,STEP,ci.last}(ts.t - shift) + if ci.first == FIRST && ci.last == LAST + return ts + else + # shift the timestep over by (ci.first = FIRST)/STEP + return FixedTimestep{ci.first,STEP,ci.last}(ts.t - Int((ci.first - FIRST)/STEP)) + end end function get_shifted_ts(ci, ts::VariableTimestep{TIMES}) where {TIMES} - idx_start = findfirst(TIMES .== ci.first) - idx_end = findfirst(TIMES .== ci.last) - shift = idx_start - 1 - - return VariableTimestep{TIMES[idx_start:idx_end]}(ts.t - shift) + if ci.first == TIMES[1] && ci.last == TIMES[end] + return ts + else + # shift the timestep over by the number of timesteps between the model first and the ts first + idx_start = findfirst(TIMES .== ci.first) + idx_finish = findfirst(TIMES .== ci.last) + return VariableTimestep{TIMES[idx_start:idx_finish]}(ts.t - idx_start - 1) + end end function run_timestep(ci::AbstractComponentInstance, clock::Clock, dims::DimValueDict) From b4b0f28190637c96bf48bf57cb519a4b826bbdc1 Mon Sep 17 00:00:00 2001 From: lrennels Date: Tue, 9 Feb 2021 09:22:32 -0700 Subject: [PATCH 18/32] Fix off by one bug --- src/core/instances.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/instances.jl b/src/core/instances.jl index 36e224f5d..2289deeef 100644 --- a/src/core/instances.jl +++ b/src/core/instances.jl @@ -294,7 +294,7 @@ function get_shifted_ts(ci, ts::VariableTimestep{TIMES}) where {TIMES} # shift the timestep over by the number of timesteps between the model first and the ts first idx_start = findfirst(TIMES .== ci.first) idx_finish = findfirst(TIMES .== ci.last) - return VariableTimestep{TIMES[idx_start:idx_finish]}(ts.t - idx_start - 1) + return VariableTimestep{TIMES[idx_start:idx_finish]}(ts.t - idx_start + 1) end end From 2c8896a739adea7e9ff153bc70e6cda337543e9a Mon Sep 17 00:00:00 2001 From: lrennels Date: Tue, 9 Feb 2021 13:14:11 -0700 Subject: [PATCH 19/32] Fix typo in test_all_models script --- contrib/test_all_models.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/test_all_models.jl b/contrib/test_all_models.jl index 224e64186..96b10d2a5 100644 --- a/contrib/test_all_models.jl +++ b/contrib/test_all_models.jl @@ -21,7 +21,7 @@ packages_to_test = [ "MimiSNEASY" => ("https://github.com/anthofflab/MimiSNEASY.jl", "master"), "MimiFAIR" => ("https://github.com/anthofflab/MimiFAIR.jl", "master"), "MimiMAGICC" => ("https://github.com/anthofflab/MimiMAGICC.jl", "master"), - "MimiHECTOR" => ("https://github.com/anthofflab/MimiHector.jl", "master") + "MimiHector" => ("https://github.com/anthofflab/MimiHector.jl", "master") ] using Pkg From 6f5db8d49a8abe0b794e3148b10da521bc3ad0f9 Mon Sep 17 00:00:00 2001 From: lrennels Date: Wed, 17 Feb 2021 00:36:26 -0800 Subject: [PATCH 20/32] Add tests for TimestepIndex and TimestepValue --- docs/src/howto/howto_4.md | 3 +- test/test_firstlast.jl | 98 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/docs/src/howto/howto_4.md b/docs/src/howto/howto_4.md index d1c1052b1..88c4ddcde 100644 --- a/docs/src/howto/howto_4.md +++ b/docs/src/howto/howto_4.md @@ -52,8 +52,7 @@ Indexing into a variable or parameter's `time` dimension with an `Integer` is de end ``` -`TimestepIndex` has one field, `index`, which refers to the absolute index in the parameter or variable array's `time` dimension. Thus, constructing a `TimestepIndex` is done by simply writing `TimestepIndex(index::Int)`. Looking back at our original component example -one could modify the first line of `run_timestep` to always refer to the first timestep of `p.d` with the following. One may index into the `time` dimension with a single `TimestepIndex`, or an `Array` of them. +`TimestepIndex` has one field, `index`, which refers to the absolute index in the parameter or variable array's `time` dimension. Thus, constructing a `TimestepIndex` is done by simply writing `TimestepIndex(index::Int)`. Looking back at our original component example, one could modify the first line of `run_timestep` to always refer to the first timestep of `p.d` with the following. One may index into the `time` dimension with a single `TimestepIndex`, or an `Array` of them. ```julia v.A[t] = p.c + p.d[TimestepIndex(1)] ``` diff --git a/test/test_firstlast.jl b/test/test_firstlast.jl index 230ff91c4..d8532baa2 100644 --- a/test/test_firstlast.jl +++ b/test/test_firstlast.jl @@ -211,4 +211,102 @@ run(m2) @test ismissing(m[:MyComp, :a][end]) @test (m[:MyComp, :a])[2:9] == (m2[:MyComp, :a])[2:9] == [-999., 2., 3., 4., 5., 6., 7., 999.] +# +# TimestepIndex and TimestepValue +# + +# TimestepIndex: Test Equality - should match up with t.t +@defcomp MyComp begin + a = Variable(index=[time]) + b = Variable(index=[time]) + function run_timestep(p, v, d, t) + v.a[t] = t == TimestepIndex(t.t) + v.b[t] = 1. + end +end + +m = Model() +set_dimension!(m, :time, collect(1:15)) +add_comp!(m, MyComp, first = 5, last = 10) +run(m) + +for i in collect(1:15) + @test m[:MyComp, :a][i] === m[:MyComp, :b][i] +end + +# TimestepValue: Test Equality - should match up with the time index +@defcomp MyComp begin + a = Variable(index=[time]) + b = Variable(index=[time]) + function run_timestep(p, v, d, t) + v.a[t] = t == TimestepValue(t.t + 4) + v.b[t] = 1. + end +end + +m = Model() +set_dimension!(m, :time, collect(1:15)) +add_comp!(m, MyComp, first = 5, last = 10) +run(m) + +for i in collect(1:15) + @test m[:MyComp, :a][i] === m[:MyComp, :b][i] +end + +# TimestepIndex: Test that Get and Set Index are Relative to Component, not Model +@defcomp MyComp begin + a = Variable(index=[time]) + b = Variable(index=[time]) + function run_timestep(p, v, d, t) + v.a[t] = t.t + v.b[TimestepIndex(t.t)] = t.t + end +end + +m = Model() +set_dimension!(m, :time, collect(1:15)) +add_comp!(m, MyComp, first = 5, last = 10) +run(m) +@test m[:MyComp, :a] == m[:MyComp, :b] + +@defcomp MyComp begin + a = Variable(index=[time]) + function run_timestep(p, v, d, t) + if t == TimestepIndex(1) + v.a[TimestepIndex(1)] = 1 + else + v.a[t] = 0 + end + end +end + +for year in collect(1995:1999) + m = Model() + set_dimension!(m, :time, collect(1995:2000)) + add_comp!(m, MyComp, first = year) + run(m) + idx = year - 1995 + 1 + @test m[:MyComp, :a][idx] == 1.0 +end + +# TimestepValue: Test that Get and Set Index are Relative to Component, not Model +@defcomp MyComp begin + a = Variable(index=[time]) + function run_timestep(p, v, d, t) + if t == TimestepValue(1999) + v.a[TimestepValue(1999)] = 1 + else + v.a[t] = 0 + end + end +end + +for year in collect(1995:1999) + m = Model() + set_dimension!(m, :time, collect(1995:2000)) + add_comp!(m, MyComp, first = year) + run(m) + @test m[:MyComp, :a][5] == 1.0 +end + end #module \ No newline at end of file From e0755cac18f19e93cb4a20b25d6bd84f42725f56 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Fri, 19 Feb 2021 12:06:31 -0800 Subject: [PATCH 21/32] Add some TODOs --- src/core/build.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core/build.jl b/src/core/build.jl index 770e7a736..e881d314a 100644 --- a/src/core/build.jl +++ b/src/core/build.jl @@ -71,6 +71,9 @@ function _instantiate_component_vars(md::ModelDef, comp_def::ComponentDef) names = Symbol[nameof(def) for def in var_defs] values = Any[_instantiate_datum(md, def) for def in var_defs] + # TODO Loop over content of values, if an element is a TimestepArray, replace it with a new + # TimestepArray that is properly shifted etc, and essentially a view into the original data + # that the TSA here had. types = DataType[_instance_datatype(md, def) for def in var_defs] paths = repeat(Any[comp_def.comp_path], length(names)) @@ -225,8 +228,11 @@ end function _instantiate_params(comp_def::ComponentDef, par_dict::Dict{Tuple{ComponentPath, Symbol}, Any}) # @info "Instantiating params for $(comp_def.comp_path)" comp_path = comp_def.comp_path - names = parameter_names(comp_def) + names = parameter_names(comp_def) vals = Any[par_dict[(comp_path, name)] for name in names] + # TODO Loop over content of vals, if an element is a TimestepArray, replace it with a new + # TimestepArray that is properly shifted etc, and essentially a view into the original data + # that the TSA here had. types = DataType[typeof(val) for val in vals] paths = repeat([comp_def.comp_path], length(names)) From 542aa030613651558e536ddf1d5e10a2f3d16507 Mon Sep 17 00:00:00 2001 From: lrennels Date: Sat, 20 Feb 2021 02:12:31 -0800 Subject: [PATCH 22/32] Add views and break everything --- src/core/build.jl | 28 +++++++--- src/core/dimensions.jl | 12 +++++ src/core/time_arrays.jl | 101 ++++++++++++++++++------------------ src/core/types/time.jl | 9 +++- test/test_components.jl | 5 +- test/test_firstlast.jl | 37 ++++++------- test/test_parametertypes.jl | 17 +++--- test/test_timesteps.jl | 12 +++-- 8 files changed, 130 insertions(+), 91 deletions(-) diff --git a/src/core/build.jl b/src/core/build.jl index e881d314a..7d40348a9 100644 --- a/src/core/build.jl +++ b/src/core/build.jl @@ -1,5 +1,25 @@ connector_comp_name(i::Int) = Symbol("ConnectorComp$i") +# helper function to substitute views for data +function _substitute_views!(vals::Array{T, N}, comp_def) where {T, N} + times = [keys(comp_def.dim_dict[:time])...] + first_idx = findfirst(times .== comp_def.first) + last_idx = findfirst(times .== comp_def.last) + for (i, val) in enumerate(vals) + # DESIGN DISCUSSION should we deep copy here or can we just do in-place val.data = _get_view(val, first_idx, last_idx) + if val isa TimestepArray + new_val = deepcopy(val) + new_val.data = _get_view(val, first_idx, last_idx) + vals[i] = new_val + end + end +end + +function _get_view(val::TimestepArray{T_TS, T, N, ti}, first_idx, last_idx) where {T_TS, T, N, ti} + idxs = Array{Any}(fill(:, N)) + idxs[ti] = first_idx:last_idx + return view(val.data, idxs...) +end # Return the datatype to use for instance variables/parameters function _instance_datatype(md::ModelDef, def::AbstractDatumDef) @@ -71,9 +91,7 @@ function _instantiate_component_vars(md::ModelDef, comp_def::ComponentDef) names = Symbol[nameof(def) for def in var_defs] values = Any[_instantiate_datum(md, def) for def in var_defs] - # TODO Loop over content of values, if an element is a TimestepArray, replace it with a new - # TimestepArray that is properly shifted etc, and essentially a view into the original data - # that the TSA here had. + _substitute_views!(values, comp_def) types = DataType[_instance_datatype(md, def) for def in var_defs] paths = repeat(Any[comp_def.comp_path], length(names)) @@ -230,9 +248,7 @@ function _instantiate_params(comp_def::ComponentDef, par_dict::Dict{Tuple{Compon comp_path = comp_def.comp_path names = parameter_names(comp_def) vals = Any[par_dict[(comp_path, name)] for name in names] - # TODO Loop over content of vals, if an element is a TimestepArray, replace it with a new - # TimestepArray that is properly shifted etc, and essentially a view into the original data - # that the TSA here had. + _substitute_views!(vals, comp_def) types = DataType[typeof(val) for val in vals] paths = repeat([comp_def.comp_path], length(names)) diff --git a/src/core/dimensions.jl b/src/core/dimensions.jl index bac53bad0..ca2f0dbd0 100644 --- a/src/core/dimensions.jl +++ b/src/core/dimensions.jl @@ -94,6 +94,18 @@ function set_dimension!(ccd::AbstractCompositeComponentDef, name::Symbol, keys:: dim = Dimension(keys) if name == :time + + # check to make sure if we are setting time dimension for the Model that + # it doesn't start after, or end before, any of the components. Note that + # here we can dependent on the invariant that all subcomponents of a composite + # component have the same first and last bounds + ccd_first = [keys...][1] + ccd_last = [keys...][end] + for subcomp in compdefs(ccd) + subcomp.first_free || ccd_first > subcomp.first && error("Top time dimension must end after or at same time as all it's subcomponents, but $(ccd_first) is after $(subcomp.first).") + subcomp.last_free || ccd_last < subcomp.last && error("Top time dimension must start before or at same time as all it's subcomponents, but $(ccd_last) is before $(subcomp.last).") + end + propagate_time!(ccd, dim) set_uniform!(ccd, isuniform(keys)) end diff --git a/src/core/time_arrays.jl b/src/core/time_arrays.jl index bc6fb78fc..e3f73c51c 100644 --- a/src/core/time_arrays.jl +++ b/src/core/time_arrays.jl @@ -168,6 +168,10 @@ end # b. TimestepVector # +function Base.length(v::TimestepVector) + return length(v.data) +end + function Base.getindex(v::TimestepVector{FixedTimestep{FIRST, STEP}, T}, ts::FixedTimestep{FIRST, STEP, LAST}) where {T, FIRST, STEP, LAST} data = v.data[ts.t] _missing_data_check(data, ts.t) @@ -179,35 +183,34 @@ function Base.getindex(v::TimestepVector{VariableTimestep{TIMES}, T}, ts::Variab end function Base.getindex(v::TimestepVector{FixedTimestep{D_FIRST, STEP}, T}, ts::FixedTimestep{T_FIRST, STEP, LAST}) where {T, D_FIRST, T_FIRST, STEP, LAST} - t = Int(ts.t + (T_FIRST - D_FIRST) / STEP) - data = v.data[t] - _missing_data_check(data, t) + data = v.data[ts.t] + _missing_data_check(data, ts.t) end function Base.getindex(v::TimestepVector{VariableTimestep{D_TIMES}, T}, ts::VariableTimestep{T_TIMES}) where {T, D_TIMES, T_TIMES} - t = ts.t + findfirst(isequal(T_TIMES[1]), D_TIMES) - 1 - data = v.data[t] - _missing_data_check(data, t) + data = v.data[ts.t] + _missing_data_check(data, ts.t) end function Base.getindex(v::TimestepVector{FixedTimestep{FIRST, STEP}, T_data}, ts::TimestepValue{T_time}) where {T_data, FIRST, STEP, T_time} LAST = FIRST + ((length(v.data)-1) * STEP) t = _get_time_value_position([FIRST:STEP:LAST...], ts) + t = t - v.data.offset1 data = v.data[t] _missing_data_check(data, t) end function Base.getindex(v::TimestepVector{VariableTimestep{TIMES}, T_data}, ts::TimestepValue{T_time}) where {T_data, TIMES, T_time} t = _get_time_value_position(TIMES, ts) + t = t - v.data.offset1 data = v.data[t] _missing_data_check(data, t) end function Base.getindex(v::TimestepVector, ts::TimestepIndex) - t = ts.index - _index_bounds_check(v.data, 1, t) - data = v.data[t] - _missing_data_check(data, t) + _index_bounds_check(v.data, 1, ts.index) + data = v.data[ts.index] + _missing_data_check(data, ts.index) end function Base.setindex!(v::TimestepVector{FixedTimestep{FIRST, STEP}, T}, val, ts::FixedTimestep{FIRST, STEP, LAST}) where {T, FIRST, STEP, LAST} @@ -219,23 +222,23 @@ function Base.setindex!(v::TimestepVector{VariableTimestep{TIMES}, T}, val, ts:: end function Base.setindex!(v::TimestepVector{FixedTimestep{D_FIRST, STEP}, T}, val, ts::FixedTimestep{T_FIRST, STEP, LAST}) where {T, D_FIRST, T_FIRST, STEP, LAST} - t = Int(ts.t + (T_FIRST - D_FIRST) / STEP) - setindex!(v.data, val, t) + setindex!(v.data, val, ts.t) end function Base.setindex!(v::TimestepVector{VariableTimestep{D_TIMES}, T}, val, ts::VariableTimestep{T_TIMES}) where {T, D_TIMES, T_TIMES} - t = ts.t + findfirst(isequal(T_TIMES[1]), D_TIMES) - 1 - setindex!(v.data, val, t) + setindex!(v.data, val, ts.t) end function Base.setindex!(v::TimestepVector{FixedTimestep{FIRST, STEP}, T_data}, val, ts::TimestepValue{T_time}) where {T_data, FIRST, STEP, T_time} LAST = FIRST + ((length(v.data)-1) * STEP) t = _get_time_value_position([FIRST:STEP:LAST...], ts) + t = t - v.data.offset1 setindex!(v.data, val, t) end function Base.setindex!(v::TimestepVector{VariableTimestep{TIMES}, T_data}, val, ts::TimestepValue{T_time}) where {T_data, TIMES, T_time} t = _get_time_value_position(TIMES, ts) + t = t - v.data.offset1 setindex!(v.data, val, t) end @@ -263,10 +266,6 @@ function Base.setindex!(v::TimestepVector{VariableTimestep{TIMES}, T}, val, i::A _throw_int_setindex_error() end -function Base.length(v::TimestepVector) - return length(v.data) -end - # # c. TimestepMatrix # @@ -282,15 +281,13 @@ function Base.getindex(mat::TimestepMatrix{VariableTimestep{TIMES}, T, 1}, ts::V end function Base.getindex(mat::TimestepMatrix{FixedTimestep{D_FIRST, STEP}, T, 1}, ts::FixedTimestep{T_FIRST, STEP, LAST}, idx::AnyIndex) where {T, D_FIRST, T_FIRST, STEP, LAST} - t = Int(ts.t + (T_FIRST - D_FIRST) / STEP) - data = mat.data[t, idx] - _missing_data_check(data, t) + data = mat.data[ts.t, idx] + _missing_data_check(data, ts.t) end function Base.getindex(mat::TimestepMatrix{VariableTimestep{D_TIMES}, T, 1}, ts::VariableTimestep{T_TIMES}, idx::AnyIndex) where {T, D_TIMES, T_TIMES} - t = ts.t + findfirst(isequal(T_TIMES[1]), D_TIMES) - 1 - data = mat.data[t, idx] - _missing_data_check(data, t) + data = mat.data[ts.t, idx] + _missing_data_check(data, ts.t) end function Base.getindex(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T, 2}, idx::AnyIndex, ts::FixedTimestep{FIRST, STEP, LAST}) where {T, FIRST, STEP, LAST} @@ -304,26 +301,26 @@ function Base.getindex(mat::TimestepMatrix{VariableTimestep{TIMES}, T, 2}, idx:: end function Base.getindex(mat::TimestepMatrix{FixedTimestep{D_FIRST, STEP}, T, 2}, idx::AnyIndex, ts::FixedTimestep{T_FIRST, STEP, LAST}) where {T, D_FIRST, T_FIRST, STEP, LAST} - t = Int(ts.t + (T_FIRST - D_FIRST) / STEP) - data = mat.data[idx, t] - _missing_data_check(data, t) + data = mat.data[idx, ts.t] + _missing_data_check(data, ts.t) end function Base.getindex(mat::TimestepMatrix{VariableTimestep{D_TIMES}, T, 2}, idx::AnyIndex, ts::VariableTimestep{T_TIMES}) where {T, D_TIMES, T_TIMES} - t = ts.t + findfirst(isequal(T_TIMES[1]), D_TIMES) - 1 - data = mat.data[idx, t] - _missing_data_check(data, t) + data = mat.data[idx, ts.t] + _missing_data_check(data, ts.t) end function Base.getindex(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T_data, 1}, ts::TimestepValue{T_time}, idx::AnyIndex) where {T_data, FIRST, STEP, T_time} LAST = FIRST + ((size(mat.data, 1) - 1) * STEP) t = _get_time_value_position([FIRST:STEP:LAST...], ts) + t = t - mat.data.indices[1][1] + 1 data = mat.data[t, idx] _missing_data_check(data, t) end function Base.getindex(mat::TimestepMatrix{VariableTimestep{TIMES}, T_data, 1}, ts::TimestepValue{T_time}, idx::AnyIndex) where {T_data, TIMES, T_time} t = _get_time_value_position(TIMES, ts) + t = t - mat.data.indices[1][1] + 1 data = mat.data[t, idx] _missing_data_check(data, t) end @@ -338,12 +335,14 @@ end function Base.getindex(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T_data, 2}, idx::AnyIndex, ts::TimestepValue{T_time}) where {T_data, FIRST, STEP, T_time} LAST = FIRST + ((size(mat.data, 2) - 1) * STEP) t = _get_time_value_position([FIRST:STEP:LAST...], ts) + t = t - mat.data.indices[2][1] + 1 data = mat.data[idx, t] _missing_data_check(data, t) end function Base.getindex(mat::TimestepMatrix{VariableTimestep{TIMES}, T_data, 2}, idx::AnyIndex, ts::TimestepValue{T_time}) where {T_data, TIMES, T_time} t = _get_time_value_position(TIMES, ts) + t = t - mat.data.indices[2][1] + 1 data = mat.data[idx, t] _missing_data_check(data, t) end @@ -364,13 +363,11 @@ function Base.setindex!(mat::TimestepMatrix{VariableTimestep{TIMES}, T, 1}, val, end function Base.setindex!(mat::TimestepMatrix{FixedTimestep{D_FIRST, STEP}, T, 1}, val, ts::FixedTimestep{T_FIRST, STEP, LAST}, idx::AnyIndex) where {T, D_FIRST, T_FIRST, STEP, LAST} - t = Int(ts.t + (T_FIRST - D_FIRST) / STEP) - setindex!(mat.data, val, t, idx) + setindex!(mat.data, val, ts.t, idx) end function Base.setindex!(mat::TimestepMatrix{VariableTimestep{D_TIMES}, T, 1}, val, ts::VariableTimestep{T_TIMES}, idx::AnyIndex) where {T, D_TIMES, T_TIMES} - t = ts.t + findfirst(isequal(T_TIMES[1]), D_TIMES) - 1 - setindex!(mat.data, val, t, idx) + setindex!(mat.data, val, ts.t, idx) end function Base.setindex!(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T, 2}, val, idx::AnyIndex, ts::FixedTimestep{FIRST, STEP, LAST}) where {T, FIRST, STEP, LAST} @@ -382,34 +379,36 @@ function Base.setindex!(mat::TimestepMatrix{VariableTimestep{TIMES}, T, 2}, val, end function Base.setindex!(mat::TimestepMatrix{FixedTimestep{D_FIRST, STEP}, T, 2}, val, idx::AnyIndex, ts::FixedTimestep{T_FIRST, STEP, LAST}) where {T, D_FIRST, T_FIRST, STEP, LAST} - t = Int(ts.t + (T_FIRST - D_FIRST) / STEP) - setindex!(mat.data, val, idx, t) + setindex!(mat.data, val, ts.t, idx) end function Base.setindex!(mat::TimestepMatrix{VariableTimestep{D_TIMES}, T, 2}, val, idx::AnyIndex, ts::VariableTimestep{T_TIMES}) where {T, D_TIMES, T_TIMES} - t = ts.t + findfirst(isequal(T_TIMES[1]), D_TIMES) - 1 - setindex!(mat.data, val, idx, t) + setindex!(mat.data, val, ts.t, idx) end function Base.setindex!(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T_data, 1}, val, ts::TimestepValue{T_time}, idx::AnyIndex) where {T_data, FIRST, STEP, T_time} LAST = FIRST + ((size(mat.data, 1) - 1) * STEP) t = _get_time_value_position([FIRST:STEP:LAST...], ts) + t = t - mat.data.indices[1][1] + 1 setindex!(mat.data, val, t, idx) end function Base.setindex!(mat::TimestepMatrix{VariableTimestep{TIMES}, T_data, 1}, val, ts::TimestepValue{T_time}, idx::AnyIndex) where {T_data, TIMES, T_time} t = _get_time_value_position(TIMES, ts) + t = t - mat.data.indices[1][1] + 1 setindex!(mat.data, val, t, idx) end function Base.setindex!(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T_data, 2}, val, idx::AnyIndex, ts::TimestepValue{T_time}) where {T_data, FIRST, STEP, T_time} LAST = FIRST + ((size(mat.data, 1) - 1) * STEP) t = _get_time_value_position([FIRST:STEP:LAST...], ts) + t = t - mat.data.indices[2][1] + 1 setindex!(mat.data, val, idx, t) end function Base.setindex!(mat::TimestepMatrix{VariableTimestep{TIMES}, T_data, 2}, val, idx::AnyIndex, ts::TimestepValue{T_time}) where {T_data, TIMES, T_time} t = _get_time_value_position(TIMES, ts) + t = t - mat.data.indices[2][1] + 1 setindex!(mat.data, val, idx, t) end @@ -509,14 +508,12 @@ end function Base.getindex(arr::TimestepArray{FixedTimestep{D_FIRST, STEP}, T, N, ti}, idxs::Union{FixedTimestep{T_FIRST, STEP, LAST}, AnyIndex}...) where {T, N, ti, D_FIRST, T_FIRST, STEP, LAST} idxs1, ts, idxs2 = split_indices(idxs, ti) - t = Int(ts.t + (FIRST - TIMES[1]) / STEP) - return arr.data[idxs1..., t, idxs2...] + return arr.data[idxs1..., ts.t, idxs2...] end function Base.getindex(arr::TimestepArray{VariableTimestep{D_TIMES}, T, N, ti}, idxs::Union{VariableTimestep{T_TIMES}, AnyIndex}...) where {T, N, ti, D_TIMES, T_TIMES} idxs1, ts, idxs2 = split_indices(idxs, ti) - t = ts.t + findfirst(isequal(T_TIMES[1]), D_TIMES) - 1 - return arr.data[idxs1..., t, idxs2...] + return arr.data[idxs1..., ts.t, idxs2...] end function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T_data, N, ti}, idxs::Union{TimestepValue{T_time}, AnyIndex}...) where {T_data, N, ti, FIRST, STEP, T_time} @@ -524,6 +521,7 @@ function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T_data, N, idxs1, ts, idxs2 = split_indices(idxs, ti) LAST = FIRST + ((size(arr.data, ti) - 1) * STEP) t = _get_time_value_position([FIRST:STEP:LAST...], ts) + t = t - arr.data.indices[ti][1] + 1 return arr.data[idxs1..., t, idxs2...] end @@ -531,6 +529,7 @@ function Base.getindex(arr::TimestepArray{VariableTimestep{TIMES}, T_data, N, ti _single_index_check(arr.data, idxs) idxs1, ts, idxs2 = split_indices(idxs, ti) t = _get_time_value_position(TIMES, ts) + t = t - arr.data.indices[ti][1] + 1 return arr.data[idxs1..., t, idxs2...] end @@ -562,14 +561,12 @@ end function Base.setindex!(arr::TimestepArray{FixedTimestep{D_FIRST, STEP}, T, N, ti}, val, idxs::Union{FixedTimestep{T_FIRST, STEP, LAST}, AnyIndex}...) where {T, N, ti, D_FIRST, T_FIRST, STEP, LAST} idxs1, ts, idxs2 = split_indices(idxs, ti) - t = ts.t + findfirst(isequal(T_FIRST[1]), D_FIRST) - 1 - setindex!(arr.data, val, idxs1..., t, idxs2...) + setindex!(arr.data, val, idxs1..., ts.t, idxs2...) end function Base.setindex!(arr::TimestepArray{VariableTimestep{D_TIMES}, T, N, ti}, val, idxs::Union{VariableTimestep{T_TIMES}, AnyIndex}...) where {T, N, ti, D_TIMES, T_TIMES} idxs1, ts, idxs2 = split_indices(idxs, ti) - t = ts.t + findfirst(isequal(T_FIRST[1]), T_TIMES) - 1 - setindex!(arr.data, val, idxs1..., t, idxs2...) + setindex!(arr.data, val, idxs1..., ts.t, idxs2...) end function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T_data, N, ti}, val, idxs::Union{TimestepValue{T_time}, AnyIndex}...) where {T_data, N, ti, FIRST, STEP, T_time} @@ -577,6 +574,7 @@ function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T_data, N idxs1, ts, idxs2 = split_indices(idxs, ti) LAST = FIRST + ((size(arr.data, ti) - 1) * STEP) t = _get_time_value_position([FIRST:STEP:LAST...], ts) + t = t - arr.data[ti][1] + 1 setindex!(arr.data, val, idxs1..., t, idxs2...) end @@ -584,19 +582,18 @@ function Base.setindex!(arr::TimestepArray{VariableTimestep{TIMES}, T_data, N, t _single_index_check(arr.data, idxs) idxs1, ts, idxs2 = split_indices(idxs, ti) t = _get_time_value_position(TIMES, ts) + t = t - arr.data[ti][1] + 1 setindex!(arr.data, val, idxs1..., t, idxs2...) end function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T, N, ti}, val, idxs::Union{TimestepIndex, AnyIndex}...) where {T, N, ti, FIRST, STEP} idxs1, ts, idxs2 = split_indices(idxs, ti) - t = ts.index - setindex!(arr.data, val, idxs1..., t, idxs2...) + setindex!(arr.data, val, idxs1..., ts.index, idxs2...) end function Base.setindex!(arr::TimestepArray{VariableTimestep{TIMES}, T, N, ti}, val, idxs::Union{TimestepIndex, AnyIndex}...) where {T, N, ti, TIMES} idxs1, ts, idxs2 = split_indices(idxs, ti) - t = ts.index - setindex!(arr.data, val, idxs1..., t, idxs2...) + setindex!(arr.data, val, idxs1..., ts.index, idxs2...) end # DEPRECATION - EVENTUALLY REMOVE @@ -632,12 +629,14 @@ function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T_data, N, idxs1, ts_array, idxs2 = split_indices(idxs, ti) LAST = FIRST + ((length(arr.data)-1) * STEP) ts_idxs = _get_ts_indices(ts_array, [FIRST:STEP:LAST...]) + ts_idxs = ts_idxs .- arr.data.indices[ti][1] + 1 return arr.data[idxs1..., ts_idxs, idxs2...] end function Base.getindex(arr::TimestepArray{VariableTimestep{TIMES}, T_data, N, ti}, idxs::Union{Array{TimestepValue{T_times},1}, AnyIndex}...) where {T_data, N, ti, TIMES, T_times} idxs1, ts_array, idxs2 = split_indices(idxs, ti) ts_idxs = _get_ts_indices(ts_array, TIMES) + ts_idxs = ts_idxs .- arr.data.indices[ti][1] + 1 return arr.data[idxs1..., ts_idxs, idxs2...] end @@ -651,12 +650,14 @@ function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T_data, N idxs1, ts_array, idxs2 = split_indices(idxs, ti) LAST = FIRST + ((length(arr.data)-1) * STEP) ts_idxs = _get_ts_indices(ts_array, [FIRST:STEP:LAST...]) + ts_idxs = ts_idxs .- arr.data.indices[ti][1] + 1 setindex!(arr.data, vals, idxs1..., ts_idxs, idxs2...) end function Base.setindex!(arr::TimestepArray{VariableTimestep{TIMES}, T_data, N, ti}, vals, idxs::Union{Array{TimestepValue{T_times},1}, AnyIndex}...) where {T_data, N, ti, TIMES, T_times} idxs1, ts_array, idxs2 = split_indices(idxs, ti) ts_idxs = _get_ts_indices(ts_array, TIMES) + ts_idxs = ts_idxs .- arr.data.indices[ti][1] + 1 setindex!(arr.data, vals, idxs1..., ts_idxs, idxs2...) end diff --git a/src/core/types/time.jl b/src/core/types/time.jl index 422f86a79..7bdde8183 100644 --- a/src/core/types/time.jl +++ b/src/core/types/time.jl @@ -60,8 +60,15 @@ mutable struct Clock{T <: AbstractTimestep} <: MimiStruct end end +# TDESIGN DISCUSSION: add this for performance? +# mutable struct TimestepArray{T_TS <: AbstractTimestep, T, N, ti, S<:AbstractArray{T,N}} <: MimiStruct +# data::S mutable struct TimestepArray{T_TS <: AbstractTimestep, T, N, ti} <: MimiStruct - data::Array{T, N} + data::Union{Array{T, N}, SubArray} + + function TimestepArray{T_TS, T, N, ti}(d::SubArray) where {T_TS, T, N, ti} + return new(d) + end function TimestepArray{T_TS, T, N, ti}(d::Array{T, N}) where {T_TS, T, N, ti} return new(d) diff --git a/test/test_components.jl b/test/test_components.jl index a654b951a..e9951bf63 100644 --- a/test/test_components.jl +++ b/test/test_components.jl @@ -137,7 +137,6 @@ ci = compinstance(m, :C) # Get the component instance m = Model() set_dimension!(m, :time, 2000:2100) - add_comp!(m, testcomp1, :C; first = 2010, last = 2090) cd = compdef(m.md, :C) # Get the component definition in the model @@ -145,14 +144,16 @@ cd = compdef(m.md, :C) # Get the component definition in the model @test cd.last == 2090 set_dimension!(m, :time, 2010:2090) - set_param!(m, :C, :par1, zeros(81)) Mimi.build!(m) # Build the model + ci = compinstance(m, :C) # Get the component instance @test ci.first == 2010 # The component instance's first and last values are the same as in the comp def @test ci.last == 2090 set_dimension!(m, :time, 2000:2200) # Reset the time dimension +update_param!(m, :par1, zeros(201)) # Have to reset the parameter to have the same width as the model time dimension + cd = compdef(m.md, :C) # Get the component definition in the model @test cd.first == 2010 # First and last values should still be the same @test cd.last == 2090 diff --git a/test/test_firstlast.jl b/test/test_firstlast.jl index d8532baa2..5ea74a152 100644 --- a/test/test_firstlast.jl +++ b/test/test_firstlast.jl @@ -73,9 +73,9 @@ run(m) # test that there are missing values in :emissions variables outside of the component's # run period, and no missing values in the :grosseconomy variables -@test ismissing(m[:emissions, :E][1]) -@test ismissing(m[:emissions, :E][20]) -@test sum(ismissing.(m[:emissions, :E][2:19])) == 0 +@test ismissing(m[:emissions, :E].parent[1]) +@test ismissing(m[:emissions, :E].parent[20]) +@test sum(ismissing.(m[:emissions, :E])) == 0 @test sum(ismissing.(m[:grosseconomy, :l])) == 0 # change the model dimension (widen it) @@ -99,9 +99,9 @@ run(m) # test that there are missing values in :emissions variables outside of the component's # run period, and no missing values in the :grosseconomy variables -@test ismissing(m[:emissions, :E][1]) -@test sum(ismissing.(m[:emissions, :E][20:21])) == 2 -@test sum(ismissing.(m[:emissions, :E][2:19])) == 0 +@test ismissing(m[:emissions, :E].parent[1]) +@test sum(ismissing.(m[:emissions, :E].parent[20:21])) == 2 +@test sum(ismissing.(m[:emissions, :E])) == 0 @test sum(ismissing.(m[:grosseconomy, :l])) == 0 # @@ -128,8 +128,8 @@ connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) run(m) # neither component should have a value for the first timestep -@test ismissing(m[:emissions, :E][1]) -@test ismissing(m[:grosseconomy, :YGROSS][1]) +@test ismissing(m[:emissions, :E].parent[1]) +@test ismissing(m[:grosseconomy, :YGROSS].parent[1]) # Test bounds - both end early @@ -154,8 +154,8 @@ connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) run(m) -@test ismissing(m[:emissions, :E][20]) -@test ismissing(m[:grosseconomy, :YGROSS][20]) +@test ismissing(m[:emissions, :E].parent[20]) +@test ismissing(m[:grosseconomy, :YGROSS].parent[20]) # # Test bounds - components starting or ending before/after the model @@ -207,9 +207,9 @@ set_dimension!(m2, :time, collect(1:10)) add_comp!(m2, MyComp, first = 2, last = 9) run(m2) -@test ismissing(m[:MyComp, :a][1]) -@test ismissing(m[:MyComp, :a][end]) -@test (m[:MyComp, :a])[2:9] == (m2[:MyComp, :a])[2:9] == [-999., 2., 3., 4., 5., 6., 7., 999.] +@test ismissing(m[:MyComp, :a].parent[1]) +@test ismissing(m[:MyComp, :a].parent[end]) +@test (m[:MyComp, :a]) == (m2[:MyComp, :a]) == [-999., 2., 3., 4., 5., 6., 7., 999.] # # TimestepIndex and TimestepValue @@ -231,7 +231,7 @@ add_comp!(m, MyComp, first = 5, last = 10) run(m) for i in collect(1:15) - @test m[:MyComp, :a][i] === m[:MyComp, :b][i] + @test m[:MyComp, :a].parent[i] === m[:MyComp, :b].parent[i] end # TimestepValue: Test Equality - should match up with the time index @@ -250,7 +250,7 @@ add_comp!(m, MyComp, first = 5, last = 10) run(m) for i in collect(1:15) - @test m[:MyComp, :a][i] === m[:MyComp, :b][i] + @test m[:MyComp, :a].parent[i] === m[:MyComp, :b].parent[i] end # TimestepIndex: Test that Get and Set Index are Relative to Component, not Model @@ -286,7 +286,8 @@ for year in collect(1995:1999) add_comp!(m, MyComp, first = year) run(m) idx = year - 1995 + 1 - @test m[:MyComp, :a][idx] == 1.0 + @test m[:MyComp, :a].parent[idx] == 1.0 + @test m[:MyComp, :a][1] == 1.0 end # TimestepValue: Test that Get and Set Index are Relative to Component, not Model @@ -306,7 +307,7 @@ for year in collect(1995:1999) set_dimension!(m, :time, collect(1995:2000)) add_comp!(m, MyComp, first = year) run(m) - @test m[:MyComp, :a][5] == 1.0 + @test m[:MyComp, :a].parent[5] == 1.0 end -end #module \ No newline at end of file +end #module diff --git a/test/test_parametertypes.jl b/test/test_parametertypes.jl index 2e6ffd7c7..101e6f78d 100644 --- a/test/test_parametertypes.jl +++ b/test/test_parametertypes.jl @@ -132,18 +132,17 @@ end # 1. Test with Fixed Timesteps m = Model() -set_dimension!(m, :time, 2000:2002) -add_comp!(m, MyComp2, first=2000, last=2002) -set_param!(m, :MyComp2, :x, [1, 2, 3]) - -set_dimension!(m, :time, 2000:2001) - -set_dimension!(m, :time, 2001:2002) +set_dimension!(m, :time, 2000:2004) +add_comp!(m, MyComp2, first=2001, last=2003) +set_param!(m, :MyComp2, :x, [1, 2, 3, 4, 5]) -update_param!(m, :x, [2, 3]) +@test_throws ErrorException set_dimension!(m, :time, 2002:2004) # model starts after component +@test_throws ErrorException set_dimension!(m, :time, 2000:2002) # model ends before component +set_dimension!(m, :time, 2001:2003) +update_param!(m, :x, [2, 3, 4]) x = external_param(m.md, :x) @test x.values isa Mimi.TimestepArray{Mimi.FixedTimestep{2001, 1, LAST} where LAST, Union{Missing,Float64}, 1} -@test x.values.data == [2., 3.] +@test x.values.data == [2., 3., 4.] run(m) @test m[:MyComp2, :y][1] == 2 # 2001 @test m[:MyComp2, :y][2] == 3 # 2002 diff --git a/test/test_timesteps.jl b/test/test_timesteps.jl index d406fc65d..987cfc3a5 100644 --- a/test/test_timesteps.jl +++ b/test/test_timesteps.jl @@ -150,16 +150,18 @@ set_param!(m, :Bar, :inputB, collect(1:length(years))) run(m) -@test length(m[:Foo, :output]) == length(years) +@test length(m[:Foo, :output].parent) == length(years) +@test length(m[:Bar, :output].parent) == length(years) + +@test length(m[:Foo, :output]) == length(first_foo:years[end]) @test length(m[:Bar, :output]) == length(years) yr_dim = Mimi.Dimension(years) idxs = yr_dim[first_foo]:yr_dim[years[end]] foo_output = m[:Foo, :output] -offset = first_foo - years[1] -for i in idxs - @test foo_output[i] == 5+(i-offset) # incorporate offset into i now because we set ts.t to match component not model +for i in 1:5 + @test foo_output[i] == 5+i end for i in 1:5 @@ -222,7 +224,7 @@ connect_param!(m2, :Foo2, :inputF, :Bar, :output) run(m2) -foo_output2 = m2[:Foo2, :output][yr_dim[first_foo]:yr_dim[years[end]]] +foo_output2 = m2[:Foo2, :output] for i in 1:6 @test foo_output2[i] == (i+5)^2 end From ab3ade7b1ee0abdb64f5c83e10757fca269bdc7c Mon Sep 17 00:00:00 2001 From: lrennels Date: Sat, 20 Feb 2021 15:19:03 -0800 Subject: [PATCH 23/32] Work on updating timesteparrays --- benchmark/getindex.jl | 3 +- src/core/build.jl | 3 +- src/core/defs.jl | 10 +- src/core/time_arrays.jl | 295 ++++++++++++------------------------ src/core/types/time.jl | 4 +- test/test_parametertypes.jl | 12 +- test/test_show.jl | 4 +- test/test_timesteparrays.jl | 86 ++++++----- 8 files changed, 163 insertions(+), 254 deletions(-) diff --git a/benchmark/getindex.jl b/benchmark/getindex.jl index 6a79f8b10..ea03878ed 100644 --- a/benchmark/getindex.jl +++ b/benchmark/getindex.jl @@ -35,7 +35,8 @@ end function _get_timesteparray_type(years, num_dims, dtype=Float64) if Mimi.isuniform(years) first, stepsize = Mimi.first_and_step(years) - T = Mimi.TimestepArray{Mimi.FixedTimestep{first, stepsize}, Union{dtype, Missing}, num_dims} + last = years[last] + T = Mimi.TimestepArray{Mimi.FixedTimestep{first, stepsize, last}, Union{dtype, Missing}, num_dims} else T = Mimi.TimestepArray{Mimi.VariableTimestep{(years...,)}, Union{dtype, Missing}, num_dims} end diff --git a/src/core/build.jl b/src/core/build.jl index 7d40348a9..c1a508c3b 100644 --- a/src/core/build.jl +++ b/src/core/build.jl @@ -38,8 +38,9 @@ function _instance_datatype(md::ModelDef, def::AbstractDatumDef) else if isuniform(md) first, stepsize = first_and_step(md) + last = last_period(md) first === nothing && @warn "_instance_datatype: first === nothing" - T = TimestepArray{FixedTimestep{first, stepsize}, Union{dtype, Missing}, num_dims, ti} + T = TimestepArray{FixedTimestep{first, stepsize, last}, Union{dtype, Missing}, num_dims, ti} else times = time_labels(md) T = TimestepArray{VariableTimestep{(times...,)}, Union{dtype, Missing}, num_dims, ti} diff --git a/src/core/defs.jl b/src/core/defs.jl index ab2d6bb6c..c8db6e026 100644 --- a/src/core/defs.jl +++ b/src/core/defs.jl @@ -529,19 +529,19 @@ function set_param!(md::ModelDef, param_name::Symbol, value; dims=nothing, ignor if num_dims == 0 values = value else - # # Use the first from the comp_def if it has it, else use the tree root (usu. a ModelDef) - # first = first_period(md, comp_def) - + # Use the first from the Model def, not the component, since we now say that the # data needs to match the dimensions of the model itself, so we need to allocate # the full time length even if we pad it with missings. first = first_period(md) - first === nothing && @warn "set_param!: first === nothing" + last = last_period(md) + last === nothing && @warn "set_param!: last === nothing" + if isuniform(md) stepsize = step_size(md) - values = TimestepArray{FixedTimestep{first, stepsize}, T, num_dims, ti}(value) + values = TimestepArray{FixedTimestep{first, stepsize, last}, T, num_dims, ti}(value) else times = time_labels(md) first_index = findfirst(isequal(first), times) diff --git a/src/core/time_arrays.jl b/src/core/time_arrays.jl index e3f73c51c..3dd0486b7 100644 --- a/src/core/time_arrays.jl +++ b/src/core/time_arrays.jl @@ -10,8 +10,10 @@ function get_timestep_array(md::ModelDef, T, N, ti, value) if isuniform(md) first, stepsize = first_and_step(md) + last = last_period(md) first === nothing && @warn "get_timestep_array: first === nothing" - return TimestepArray{FixedTimestep{first, stepsize}, T, N, ti}(value) + last === nothing && @warn "get_timestep_array: last === nothing" + return TimestepArray{FixedTimestep{first, stepsize, last}, T, N, ti}(value) else TIMES = (time_labels(md)...,) return TimestepArray{VariableTimestep{TIMES}, T, N, ti}(value) @@ -111,6 +113,10 @@ function _get_ts_indices(ts_array::Array{TimestepValue{T}, 1}, times::Union{Tupl return [_get_time_value_position(times, ts) for ts in ts_array] end +function _get_ts_indices(arr::Array{Colon, N}) where N + return [arr...] +end + # Base.firstindex and Base.lastindex function Base.firstindex(arr::TimestepArray{T_TS, T, N, ti}) where {T_TS, T, N, ti} if ti == 1 @@ -172,37 +178,28 @@ function Base.length(v::TimestepVector) return length(v.data) end -function Base.getindex(v::TimestepVector{FixedTimestep{FIRST, STEP}, T}, ts::FixedTimestep{FIRST, STEP, LAST}) where {T, FIRST, STEP, LAST} - data = v.data[ts.t] - _missing_data_check(data, ts.t) -end - -function Base.getindex(v::TimestepVector{VariableTimestep{TIMES}, T}, ts::VariableTimestep{TIMES}) where {T, TIMES} +function Base.getindex(v::TimestepVector, ts::FixedTimestep{FIRST, STEP, LAST}) where {FIRST, STEP, LAST} data = v.data[ts.t] _missing_data_check(data, ts.t) end -function Base.getindex(v::TimestepVector{FixedTimestep{D_FIRST, STEP}, T}, ts::FixedTimestep{T_FIRST, STEP, LAST}) where {T, D_FIRST, T_FIRST, STEP, LAST} +function Base.getindex(v::TimestepVector, ts::VariableTimestep{TIMES}) where {TIMES} data = v.data[ts.t] _missing_data_check(data, ts.t) end -function Base.getindex(v::TimestepVector{VariableTimestep{D_TIMES}, T}, ts::VariableTimestep{T_TIMES}) where {T, D_TIMES, T_TIMES} - data = v.data[ts.t] - _missing_data_check(data, ts.t) -end - -function Base.getindex(v::TimestepVector{FixedTimestep{FIRST, STEP}, T_data}, ts::TimestepValue{T_time}) where {T_data, FIRST, STEP, T_time} - LAST = FIRST + ((length(v.data)-1) * STEP) +function Base.getindex(v::TimestepVector{FixedTimestep{FIRST, STEP, LAST}, T_data}, ts::TimestepValue{T_time}) where {T_data, FIRST, STEP, LAST, T_time} t = _get_time_value_position([FIRST:STEP:LAST...], ts) - t = t - v.data.offset1 + v.data isa SubArray ? view_offset = v.data.offset1 : view_offset = 0 + t = t - view_offset data = v.data[t] _missing_data_check(data, t) end function Base.getindex(v::TimestepVector{VariableTimestep{TIMES}, T_data}, ts::TimestepValue{T_time}) where {T_data, TIMES, T_time} t = _get_time_value_position(TIMES, ts) - t = t - v.data.offset1 + v.data isa SubArray ? view_offset = v.data.offset1 : view_offset = 0 + t = t - view_offset data = v.data[t] _missing_data_check(data, t) end @@ -213,32 +210,25 @@ function Base.getindex(v::TimestepVector, ts::TimestepIndex) _missing_data_check(data, ts.index) end -function Base.setindex!(v::TimestepVector{FixedTimestep{FIRST, STEP}, T}, val, ts::FixedTimestep{FIRST, STEP, LAST}) where {T, FIRST, STEP, LAST} - setindex!(v.data, val, ts.t) -end - -function Base.setindex!(v::TimestepVector{VariableTimestep{TIMES}, T}, val, ts::VariableTimestep{TIMES}) where {T, TIMES} - setindex!(v.data, val, ts.t) -end - -function Base.setindex!(v::TimestepVector{FixedTimestep{D_FIRST, STEP}, T}, val, ts::FixedTimestep{T_FIRST, STEP, LAST}) where {T, D_FIRST, T_FIRST, STEP, LAST} +function Base.setindex!(v::TimestepVector, val, ts::FixedTimestep{FIRST, STEP, LAST}) where {FIRST, STEP, LAST} setindex!(v.data, val, ts.t) end -function Base.setindex!(v::TimestepVector{VariableTimestep{D_TIMES}, T}, val, ts::VariableTimestep{T_TIMES}) where {T, D_TIMES, T_TIMES} +function Base.setindex!(v::TimestepVector, val, ts::VariableTimestep{TIMES}) where {TIMES} setindex!(v.data, val, ts.t) end -function Base.setindex!(v::TimestepVector{FixedTimestep{FIRST, STEP}, T_data}, val, ts::TimestepValue{T_time}) where {T_data, FIRST, STEP, T_time} - LAST = FIRST + ((length(v.data)-1) * STEP) +function Base.setindex!(v::TimestepVector{FixedTimestep{FIRST, STEP, LAST}, T_data}, val, ts::TimestepValue{T_time}) where {T_data, FIRST, STEP, LAST, T_time} t = _get_time_value_position([FIRST:STEP:LAST...], ts) - t = t - v.data.offset1 + v.data isa SubArray ? view_offset = v.data.offset1 : view_offset = 0 + t = t - view_offset setindex!(v.data, val, t) end function Base.setindex!(v::TimestepVector{VariableTimestep{TIMES}, T_data}, val, ts::TimestepValue{T_time}) where {T_data, TIMES, T_time} t = _get_time_value_position(TIMES, ts) - t = t - v.data.offset1 + v.data isa SubArray ? view_offset = v.data.offset1 : view_offset = 0 + t = t - view_offset setindex!(v.data, val, t) end @@ -250,19 +240,11 @@ end # int indexing version supports old-style components and internal functions, not # part of the public API - function Base.getindex(v::TimestepVector{FixedTimestep{FIRST, STEP}, T}, i::AnyIndex_NonColon) where {T, FIRST, STEP} + function Base.getindex(v::TimestepVector, i::AnyIndex_NonColon) _throw_int_getindex_error() end -function Base.getindex(v::TimestepVector{VariableTimestep{TIMES}, T}, i::AnyIndex_NonColon) where {T, TIMES} - _throw_int_getindex_error() -end - -function Base.setindex!(v::TimestepVector{FixedTimestep{Start, STEP}, T}, val, i::AnyIndex_NonColon) where {T, Start, STEP} - _throw_int_setindex_error() -end - -function Base.setindex!(v::TimestepVector{VariableTimestep{TIMES}, T}, val, i::AnyIndex_NonColon) where {T, TIMES} +function Base.setindex!(v::TimestepVector, val, i::AnyIndex_NonColon) _throw_int_setindex_error() end @@ -270,57 +252,38 @@ end # c. TimestepMatrix # -function Base.getindex(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T, 1}, ts::FixedTimestep{FIRST, STEP, LAST}, idx::AnyIndex) where {T, FIRST, STEP, LAST} +function Base.getindex(mat::TimestepMatrix{FixedTimestep{M_FIRST, M_STEP, M_LAST}, T, 1}, ts::FixedTimestep{T_FIRST, T_STEP, T_LAST}, idx::AnyIndex) where {T, M_FIRST, M_STEP, M_LAST, T_FIRST, T_STEP, T_LAST} data = mat.data[ts.t, idx] _missing_data_check(data, ts.t) end -function Base.getindex(mat::TimestepMatrix{VariableTimestep{TIMES}, T, 1}, ts::VariableTimestep{TIMES}, idx::AnyIndex) where {T, TIMES} +function Base.getindex(mat::TimestepMatrix{VariableTimestep{M_TIMES}, T, 1}, ts::VariableTimestep{T_TIMES}, idx::AnyIndex) where {T, M_TIMES, T_TIMES} data = mat.data[ts.t, idx] _missing_data_check(data, ts.t) end -function Base.getindex(mat::TimestepMatrix{FixedTimestep{D_FIRST, STEP}, T, 1}, ts::FixedTimestep{T_FIRST, STEP, LAST}, idx::AnyIndex) where {T, D_FIRST, T_FIRST, STEP, LAST} - data = mat.data[ts.t, idx] - _missing_data_check(data, ts.t) -end - -function Base.getindex(mat::TimestepMatrix{VariableTimestep{D_TIMES}, T, 1}, ts::VariableTimestep{T_TIMES}, idx::AnyIndex) where {T, D_TIMES, T_TIMES} - data = mat.data[ts.t, idx] - _missing_data_check(data, ts.t) -end - -function Base.getindex(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T, 2}, idx::AnyIndex, ts::FixedTimestep{FIRST, STEP, LAST}) where {T, FIRST, STEP, LAST} +function Base.getindex(mat::TimestepMatrix{FixedTimestep{M_FIRST, M_STEP, M_LAST}, T, 2}, idx::AnyIndex, ts::FixedTimestep{T_FIRST, T_STEP, T_LAST}) where {T, M_FIRST, M_STEP, M_LAST, T_FIRST, T_STEP, T_LAST} data = mat.data[idx, ts.t] _missing_data_check(data, ts.t) end -function Base.getindex(mat::TimestepMatrix{VariableTimestep{TIMES}, T, 2}, idx::AnyIndex, ts::VariableTimestep{TIMES}) where {T, TIMES} +function Base.getindex(mat::TimestepMatrix{VariableTimestep{M_TIMES}, T, 2}, idx::AnyIndex, ts::VariableTimestep{T_TIMES}) where {T, M_TIMES, T_TIMES} data = mat.data[idx, ts.t] _missing_data_check(data, ts.t) end -function Base.getindex(mat::TimestepMatrix{FixedTimestep{D_FIRST, STEP}, T, 2}, idx::AnyIndex, ts::FixedTimestep{T_FIRST, STEP, LAST}) where {T, D_FIRST, T_FIRST, STEP, LAST} - data = mat.data[idx, ts.t] - _missing_data_check(data, ts.t) -end - -function Base.getindex(mat::TimestepMatrix{VariableTimestep{D_TIMES}, T, 2}, idx::AnyIndex, ts::VariableTimestep{T_TIMES}) where {T, D_TIMES, T_TIMES} - data = mat.data[idx, ts.t] - _missing_data_check(data, ts.t) -end - -function Base.getindex(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T_data, 1}, ts::TimestepValue{T_time}, idx::AnyIndex) where {T_data, FIRST, STEP, T_time} - LAST = FIRST + ((size(mat.data, 1) - 1) * STEP) +function Base.getindex(mat::TimestepMatrix{FixedTimestep{FIRST, STEP, LAST}, T_data, 1}, ts::TimestepValue{T_time}, idx::AnyIndex) where {T_data, FIRST, STEP, LAST, T_time} t = _get_time_value_position([FIRST:STEP:LAST...], ts) - t = t - mat.data.indices[1][1] + 1 + mat.data isa SubArray ? view_offset = mat.data.indices[1][1] + 1 : view_offset = 0 + t = t - view_offset data = mat.data[t, idx] _missing_data_check(data, t) end function Base.getindex(mat::TimestepMatrix{VariableTimestep{TIMES}, T_data, 1}, ts::TimestepValue{T_time}, idx::AnyIndex) where {T_data, TIMES, T_time} t = _get_time_value_position(TIMES, ts) - t = t - mat.data.indices[1][1] + 1 + mat.data isa SubArray ? view_offset = mat.data.indices[1][1] + 1 : view_offset = 0 + t = t - view_offset data = mat.data[t, idx] _missing_data_check(data, t) end @@ -332,17 +295,18 @@ function Base.getindex(mat::TimestepMatrix, ts::TimestepIndex, idx::AnyIndex) _missing_data_check(data, t) end -function Base.getindex(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T_data, 2}, idx::AnyIndex, ts::TimestepValue{T_time}) where {T_data, FIRST, STEP, T_time} - LAST = FIRST + ((size(mat.data, 2) - 1) * STEP) +function Base.getindex(mat::TimestepMatrix{FixedTimestep{FIRST, STEP, LAST}, T_data, 2}, idx::AnyIndex, ts::TimestepValue{T_time}) where {T_data, FIRST, STEP, LAST, T_time} t = _get_time_value_position([FIRST:STEP:LAST...], ts) - t = t - mat.data.indices[2][1] + 1 + mat.data isa SubArray ? view_offset = mat.data.indices[2][1] + 1 : view_offset = 0 + t = t - view_offset data = mat.data[idx, t] _missing_data_check(data, t) end function Base.getindex(mat::TimestepMatrix{VariableTimestep{TIMES}, T_data, 2}, idx::AnyIndex, ts::TimestepValue{T_time}) where {T_data, TIMES, T_time} t = _get_time_value_position(TIMES, ts) - t = t - mat.data.indices[2][1] + 1 + mat.data isa SubArray ? view_offset = mat.data.indices[2][1] + 1 : view_offset = 0 + t = t - view_offset data = mat.data[idx, t] _missing_data_check(data, t) end @@ -354,61 +318,47 @@ function Base.getindex(mat::TimestepMatrix, idx::AnyIndex, ts::TimestepIndex) _missing_data_check(data, t) end -function Base.setindex!(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T, 1}, val, ts::FixedTimestep{FIRST, STEP, LAST}, idx::AnyIndex) where {T, FIRST, STEP, LAST} - setindex!(mat.data, val, ts.t, idx) -end - -function Base.setindex!(mat::TimestepMatrix{VariableTimestep{TIMES}, T, 1}, val, ts::VariableTimestep{TIMES}, idx::AnyIndex) where {T, TIMES} - setindex!(mat.data, val, ts.t, idx) -end - -function Base.setindex!(mat::TimestepMatrix{FixedTimestep{D_FIRST, STEP}, T, 1}, val, ts::FixedTimestep{T_FIRST, STEP, LAST}, idx::AnyIndex) where {T, D_FIRST, T_FIRST, STEP, LAST} +function Base.setindex!(mat::TimestepMatrix{FixedTimestep{M_FIRST, M_STEP, M_LAST}, T, 1}, val, ts::FixedTimestep{T_FIRST, T_STEP, T_LAST}, idx::AnyIndex) where {T, M_FIRST, M_STEP, M_LAST, T_FIRST, T_STEP, T_LAST} setindex!(mat.data, val, ts.t, idx) end -function Base.setindex!(mat::TimestepMatrix{VariableTimestep{D_TIMES}, T, 1}, val, ts::VariableTimestep{T_TIMES}, idx::AnyIndex) where {T, D_TIMES, T_TIMES} +function Base.setindex!(mat::TimestepMatrix{VariableTimestep{M_TIMES}, T, 1}, val, ts::VariableTimestep{T_TIMES}, idx::AnyIndex) where {T, M_TIMES, T_TIMES} setindex!(mat.data, val, ts.t, idx) end -function Base.setindex!(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T, 2}, val, idx::AnyIndex, ts::FixedTimestep{FIRST, STEP, LAST}) where {T, FIRST, STEP, LAST} +function Base.setindex!(mat::TimestepMatrix{FixedTimestep{M_FIRST, M_STEP, M_LAST}, T, 2}, val, idx::AnyIndex, ts::FixedTimestep{T_FIRST, T_STEP, T_LAST}) where {T, M_FIRST, M_STEP, M_LAST, T_FIRST, T_STEP, T_LAST} setindex!(mat.data, val, idx, ts.t) end -function Base.setindex!(mat::TimestepMatrix{VariableTimestep{TIMES}, T, 2}, val, idx::AnyIndex, ts::VariableTimestep{TIMES}) where {T, TIMES} +function Base.setindex!(mat::TimestepMatrix{VariableTimestep{M_TIMES}, T, 2}, val, idx::AnyIndex, ts::VariableTimestep{T_TIMES}) where {T, M_TIMES, T_TIMES} setindex!(mat.data, val, idx, ts.t) end -function Base.setindex!(mat::TimestepMatrix{FixedTimestep{D_FIRST, STEP}, T, 2}, val, idx::AnyIndex, ts::FixedTimestep{T_FIRST, STEP, LAST}) where {T, D_FIRST, T_FIRST, STEP, LAST} - setindex!(mat.data, val, ts.t, idx) -end - -function Base.setindex!(mat::TimestepMatrix{VariableTimestep{D_TIMES}, T, 2}, val, idx::AnyIndex, ts::VariableTimestep{T_TIMES}) where {T, D_TIMES, T_TIMES} - setindex!(mat.data, val, ts.t, idx) -end - -function Base.setindex!(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T_data, 1}, val, ts::TimestepValue{T_time}, idx::AnyIndex) where {T_data, FIRST, STEP, T_time} - LAST = FIRST + ((size(mat.data, 1) - 1) * STEP) +function Base.setindex!(mat::TimestepMatrix{FixedTimestep{FIRST, STEP, LAST}, T_data, 1}, val, ts::TimestepValue{T_time}, idx::AnyIndex) where {T_data, FIRST, STEP, LAST, T_time} t = _get_time_value_position([FIRST:STEP:LAST...], ts) - t = t - mat.data.indices[1][1] + 1 + mat.data isa SubArray ? view_offset = mat.data.indices[1][1] + 1 : view_offset = 0 + t = t - view_offset setindex!(mat.data, val, t, idx) end function Base.setindex!(mat::TimestepMatrix{VariableTimestep{TIMES}, T_data, 1}, val, ts::TimestepValue{T_time}, idx::AnyIndex) where {T_data, TIMES, T_time} t = _get_time_value_position(TIMES, ts) - t = t - mat.data.indices[1][1] + 1 + mat.data isa SubArray ? view_offset = mat.data.indices[1][1] + 1 : view_offset = 0 + t = t - view_offset setindex!(mat.data, val, t, idx) end -function Base.setindex!(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T_data, 2}, val, idx::AnyIndex, ts::TimestepValue{T_time}) where {T_data, FIRST, STEP, T_time} - LAST = FIRST + ((size(mat.data, 1) - 1) * STEP) +function Base.setindex!(mat::TimestepMatrix{FixedTimestep{FIRST, STEP, LAST}, T_data, 2}, val, idx::AnyIndex, ts::TimestepValue{T_time}) where {T_data, FIRST, STEP, LAST, T_time} t = _get_time_value_position([FIRST:STEP:LAST...], ts) - t = t - mat.data.indices[2][1] + 1 + mat.data isa SubArray ? view_offset = mat.data.indices[2][1] + 1 : view_offset = 0 + t = t - view_offset setindex!(mat.data, val, idx, t) end function Base.setindex!(mat::TimestepMatrix{VariableTimestep{TIMES}, T_data, 2}, val, idx::AnyIndex, ts::TimestepValue{T_time}) where {T_data, TIMES, T_time} t = _get_time_value_position(TIMES, ts) - t = t - mat.data.indices[2][1] + 1 + mat.data isa SubArray ? view_offset = mat.data.indices[2][1] + 1 : view_offset = 0 + t = t - view_offset setindex!(mat.data, val, idx, t) end @@ -424,29 +374,14 @@ end # int indexing version supports old-style components and internal functions, not # part of the public API -function Base.getindex(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T, ti}, idx1::AnyIndex_NonColon, idx2::AnyIndex_NonColon) where {T, FIRST, STEP, ti} +function Base.getindex(mat::TimestepMatrix, idx1::AnyIndex_NonColon, idx2::AnyIndex_NonColon) _throw_int_getindex_error() end -function Base.getindex(mat::TimestepMatrix{VariableTimestep{TIMES}, T, ti}, idx1::AnyIndex_NonColon, idx2::AnyIndex_NonColon) where {T, TIMES, ti} - _throw_int_getindex_error() -end - -function Base.setindex!(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T, ti}, val, idx1::Int, idx2::Int) where {T, FIRST, STEP, ti} - _throw_int_setindex_error() -end - -function Base.setindex!(mat::TimestepMatrix{FixedTimestep{FIRST, STEP}, T, ti}, val, idx1::AnyIndex_NonColon, idx2::AnyIndex_NonColon) where {T, FIRST, STEP, ti} +function Base.setindex!(mat::TimestepMatrix, val, idx1::Int, idx2::Int) _throw_int_setindex_error() end -function Base.setindex!(mat::TimestepMatrix{VariableTimestep{TIMES}, T, ti}, val, idx1::Int, idx2::Int) where {T, TIMES, ti} - _throw_int_setindex_error() -end - -function Base.setindex!(mat::TimestepMatrix{VariableTimestep{TIMES}, T, ti}, val, idx1::AnyIndex_NonColon, idx2::AnyIndex_NonColon) where {T, TIMES, ti} - _throw_int_setindex_error() -end # # TimestepArray methods @@ -480,48 +415,33 @@ Base.ndims(obj::TimestepArray{T_ts, T, N, ti}) where {T_ts, T, N, ti} = N Base.eltype(obj::TimestepArray{T_ts, T, N, ti}) where {T_ts, T, N, ti} = T -first_period(obj::TimestepArray{FixedTimestep{FIRST,STEP}, T, N, ti}) where {FIRST, STEP, T, N, ti} = FIRST +first_period(obj::TimestepArray{FixedTimestep{FIRST,STEP, LAST}, T, N, ti}) where {FIRST, STEP, LAST, T, N, ti} = FIRST first_period(obj::TimestepArray{VariableTimestep{TIMES}, T, N, ti}) where {TIMES, T, N, ti} = TIMES[1] -last_period(obj::TimestepArray{FixedTimestep{FIRST, STEP}, T, N, ti}) where {FIRST, STEP, T, N, ti} = (FIRST + (size(obj, 1) - 1) * STEP) +last_period(obj::TimestepArray{FixedTimestep{FIRST, STEP, LAST}, T, N, ti}) where {FIRST, STEP, LAST, T, N, ti} = (FIRST + (size(obj, 1) - 1) * STEP) last_period(obj::TimestepArray{VariableTimestep{TIMES}, T, N, ti}) where {TIMES, T, N, ti} = TIMES[end] -time_labels(obj::TimestepArray{FixedTimestep{FIRST, STEP}, T, N, ti}) where {FIRST, STEP, T, N, ti} = collect(FIRST:STEP:(FIRST + (size(obj, 1) - 1) * STEP)) +time_labels(obj::TimestepArray{FixedTimestep{FIRST, STEP, LAST}, T, N, ti}) where {FIRST, STEP, LAST, T, N, ti} = collect(FIRST:STEP:(FIRST + (size(obj, 1) - 1) * STEP)) time_labels(obj::TimestepArray{VariableTimestep{TIMES}, T, N, ti}) where {TIMES, T, N, ti} = collect(TIMES) split_indices(idxs, ti) = idxs[1:ti - 1], idxs[ti], idxs[ti + 1:end] -function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T, N, ti}, idxs::Union{FixedTimestep{FIRST, STEP, LAST}, AnyIndex}...) where {T, N, ti, FIRST, STEP, LAST} - idxs1, ts, idxs2 = split_indices(idxs, ti) - return arr.data[idxs1..., ts.t, idxs2...] -end - -function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP, LAST}, T, N, ti}, idxs::Union{FixedTimestep{FIRST, STEP, LAST}, AnyIndex}...) where {T, N, ti, FIRST, STEP, LAST} +function Base.getindex(arr::TimestepArray{FixedTimestep{A_FIRST, A_STEP, A_LAST}, T, N, ti}, idxs::Union{FixedTimestep{T_FIRST, T_STEP, T_LAST}, AnyIndex}...) where {A_FIRST, A_STEP, A_LAST, T_FIRST, T_STEP, T_LAST, T, N, ti} idxs1, ts, idxs2 = split_indices(idxs, ti) return arr.data[idxs1..., ts.t, idxs2...] end -function Base.getindex(arr::TimestepArray{VariableTimestep{TIMES}, T, N, ti}, idxs::Union{VariableTimestep{TIMES}, AnyIndex}...) where {T, N, ti, TIMES} +function Base.getindex(arr::TimestepArray{VariableTimestep{A_TIMES}, T, N, ti}, idxs::Union{VariableTimestep{T_TIMES}, AnyIndex}...) where {A_TIMES, T_TIMES, T, N, ti} idxs1, ts, idxs2 = split_indices(idxs, ti) return arr.data[idxs1..., ts.t, idxs2...] end -function Base.getindex(arr::TimestepArray{FixedTimestep{D_FIRST, STEP}, T, N, ti}, idxs::Union{FixedTimestep{T_FIRST, STEP, LAST}, AnyIndex}...) where {T, N, ti, D_FIRST, T_FIRST, STEP, LAST} - idxs1, ts, idxs2 = split_indices(idxs, ti) - return arr.data[idxs1..., ts.t, idxs2...] -end - -function Base.getindex(arr::TimestepArray{VariableTimestep{D_TIMES}, T, N, ti}, idxs::Union{VariableTimestep{T_TIMES}, AnyIndex}...) where {T, N, ti, D_TIMES, T_TIMES} - idxs1, ts, idxs2 = split_indices(idxs, ti) - return arr.data[idxs1..., ts.t, idxs2...] -end - -function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T_data, N, ti}, idxs::Union{TimestepValue{T_time}, AnyIndex}...) where {T_data, N, ti, FIRST, STEP, T_time} +function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP, LAST}, T_data, N, ti}, idxs::Union{TimestepValue{T_time}, AnyIndex}...) where {T_data, N, ti, FIRST, STEP, LAST, T_time} _single_index_check(arr.data, idxs) idxs1, ts, idxs2 = split_indices(idxs, ti) - LAST = FIRST + ((size(arr.data, ti) - 1) * STEP) t = _get_time_value_position([FIRST:STEP:LAST...], ts) - t = t - arr.data.indices[ti][1] + 1 + arr.data isa SubArray ? view_offset = arr.data.indices[ti][1] + 1 : view_offset = 0 + t = t - view_offset return arr.data[idxs1..., t, idxs2...] end @@ -529,11 +449,12 @@ function Base.getindex(arr::TimestepArray{VariableTimestep{TIMES}, T_data, N, ti _single_index_check(arr.data, idxs) idxs1, ts, idxs2 = split_indices(idxs, ti) t = _get_time_value_position(TIMES, ts) - t = t - arr.data.indices[ti][1] + 1 + arr.data isa SubArray ? view_offset = arr.data.indices[ti][1] + 1 : view_offset = 0 + t = t - view_offset return arr.data[idxs1..., t, idxs2...] end -function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T, N, ti}, idxs::Union{TimestepIndex, AnyIndex}...) where {T, N, ti, FIRST, STEP} +function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP, LAST}, T, N, ti}, idxs::Union{TimestepIndex, AnyIndex}...) where {T, N, ti, FIRST, STEP, LAST} _single_index_check(arr.data, idxs) idxs1, ts, idxs2 = split_indices(idxs, ti) t = ts.index @@ -549,32 +470,22 @@ function Base.getindex(arr::TimestepArray{VariableTimestep{TIMES}, T, N, ti}, id return arr.data[idxs1..., t, idxs2...] end -function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T, N, ti}, val, idxs::Union{FixedTimestep{FIRST, STEP, LAST}, AnyIndex}...) where {T, N, ti, FIRST, STEP, LAST} +function Base.setindex!(arr::TimestepArray{FixedTimestep{A_FIRST, A_STEP, A_LAST}, T, N, ti}, val, idxs::Union{FixedTimestep{T_FIRST, T_STEP, T_LAST}, AnyIndex}...) where {A_FIRST, A_STEP, A_LAST, T_FIRST, T_STEP, T_LAST, T, N, ti} idxs1, ts, idxs2 = split_indices(idxs, ti) setindex!(arr.data, val, idxs1..., ts.t, idxs2...) end -function Base.setindex!(arr::TimestepArray{VariableTimestep{TIMES}, T, N, ti}, val, idxs::Union{VariableTimestep{TIMES}, AnyIndex}...) where {T, N, ti, TIMES} +function Base.setindex!(arr::TimestepArray{VariableTimestep{A_TIMES}, T, N, ti}, val, idxs::Union{VariableTimestep{T_TIMES}, AnyIndex}...) where {A_TIMES, T_TIMES, T, N, ti} idxs1, ts, idxs2 = split_indices(idxs, ti) setindex!(arr.data, val, idxs1..., ts.t, idxs2...) end -function Base.setindex!(arr::TimestepArray{FixedTimestep{D_FIRST, STEP}, T, N, ti}, val, idxs::Union{FixedTimestep{T_FIRST, STEP, LAST}, AnyIndex}...) where {T, N, ti, D_FIRST, T_FIRST, STEP, LAST} - idxs1, ts, idxs2 = split_indices(idxs, ti) - setindex!(arr.data, val, idxs1..., ts.t, idxs2...) -end - -function Base.setindex!(arr::TimestepArray{VariableTimestep{D_TIMES}, T, N, ti}, val, idxs::Union{VariableTimestep{T_TIMES}, AnyIndex}...) where {T, N, ti, D_TIMES, T_TIMES} - idxs1, ts, idxs2 = split_indices(idxs, ti) - setindex!(arr.data, val, idxs1..., ts.t, idxs2...) -end - -function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T_data, N, ti}, val, idxs::Union{TimestepValue{T_time}, AnyIndex}...) where {T_data, N, ti, FIRST, STEP, T_time} +function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP, LAST}, T_data, N, ti}, val, idxs::Union{TimestepValue{T_time}, AnyIndex}...) where {T_data, N, ti, FIRST, STEP, LAST, T_time} _single_index_check(arr.data, idxs) idxs1, ts, idxs2 = split_indices(idxs, ti) - LAST = FIRST + ((size(arr.data, ti) - 1) * STEP) t = _get_time_value_position([FIRST:STEP:LAST...], ts) - t = t - arr.data[ti][1] + 1 + arr.data isa SubArray ? view_offset = arr.data.indices[ti][1] + 1 : view_offset = 0 + t = t - view_offset setindex!(arr.data, val, idxs1..., t, idxs2...) end @@ -582,23 +493,24 @@ function Base.setindex!(arr::TimestepArray{VariableTimestep{TIMES}, T_data, N, t _single_index_check(arr.data, idxs) idxs1, ts, idxs2 = split_indices(idxs, ti) t = _get_time_value_position(TIMES, ts) - t = t - arr.data[ti][1] + 1 + arr.data isa SubArray ? view_offset = arr.data.indices[ti][1] + 1 : view_offset = 0 + t = t - view_offset setindex!(arr.data, val, idxs1..., t, idxs2...) end -function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T, N, ti}, val, idxs::Union{TimestepIndex, AnyIndex}...) where {T, N, ti, FIRST, STEP} +function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP, LAST}, T, N, ti}, val, idxs::Union{TimestepIndex, AnyIndex}...) where {FIRST, STEP, LAST, T, N, ti} idxs1, ts, idxs2 = split_indices(idxs, ti) setindex!(arr.data, val, idxs1..., ts.index, idxs2...) end -function Base.setindex!(arr::TimestepArray{VariableTimestep{TIMES}, T, N, ti}, val, idxs::Union{TimestepIndex, AnyIndex}...) where {T, N, ti, TIMES} +function Base.setindex!(arr::TimestepArray{VariableTimestep{TIMES}, T, N, ti}, val, idxs::Union{TimestepIndex, AnyIndex}...) where {TIMES, T, N, ti} idxs1, ts, idxs2 = split_indices(idxs, ti) setindex!(arr.data, val, idxs1..., ts.index, idxs2...) end # DEPRECATION - EVENTUALLY REMOVE # Colon support - this allows the time dimension to be indexed with a colon - function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T, N, ti}, idxs::AnyIndex...) where {FIRST, STEP, T, N, ti} + function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP, LAST}, T, N, ti}, idxs::AnyIndex...) where {FIRST, STEP, LAST, T, N, ti} isa(idxs[ti], AnyIndex_NonColon) ? _throw_int_getindex_error() : nothing return arr.data[idxs...] end @@ -608,7 +520,7 @@ function Base.getindex(arr::TimestepArray{VariableTimestep{TIMES}, T, N, ti}, id return arr.data[idxs...] end -function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T, N, ti}, val, idxs::AnyIndex...) where {FIRST, STEP, T, N, ti} +function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP, LAST}, T, N, ti}, val, idxs::AnyIndex...) where {FIRST, STEP, LAST, T, N, ti} isa(idxs[ti], AnyIndex_NonColon) ? _throw_int_setindex_error() : nothing setindex!(arr.data, val, idxs...) end @@ -625,18 +537,19 @@ function Base.getindex(arr::TimestepArray{TS, T, N, ti}, idxs::Union{Array{Times return arr.data[idxs1..., ts_idxs, idxs2...] end -function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T_data, N, ti}, idxs::Union{Array{TimestepValue{T_time},1}, AnyIndex}...) where {T_data, N, ti, FIRST, STEP, T_time} +function Base.getindex(arr::TimestepArray{FixedTimestep{FIRST, STEP, LAST}, T_data, N, ti}, idxs::Union{Array{TimestepValue{T_time},1}, AnyIndex}...) where {T_data, N, ti, FIRST, STEP, LAST, T_time} idxs1, ts_array, idxs2 = split_indices(idxs, ti) - LAST = FIRST + ((length(arr.data)-1) * STEP) ts_idxs = _get_ts_indices(ts_array, [FIRST:STEP:LAST...]) - ts_idxs = ts_idxs .- arr.data.indices[ti][1] + 1 + arr.data isa SubArray ? view_offset = arr.data.indices[ti][1] + 1 : view_offset = 0 + ts_idxs = ts_idxs .- view_offset return arr.data[idxs1..., ts_idxs, idxs2...] end -function Base.getindex(arr::TimestepArray{VariableTimestep{TIMES}, T_data, N, ti}, idxs::Union{Array{TimestepValue{T_times},1}, AnyIndex}...) where {T_data, N, ti, TIMES, T_times} +function Base.getindex(arr::TimestepArray{VariableTimestep{TIMES}, T_data, N, ti}, idxs::Union{Array{TimestepValue{T_time},1}, AnyIndex}...) where {T_data, N, ti, TIMES, T_time} idxs1, ts_array, idxs2 = split_indices(idxs, ti) ts_idxs = _get_ts_indices(ts_array, TIMES) - ts_idxs = ts_idxs .- arr.data.indices[ti][1] + 1 + arr.data isa SubArray ? view_offset = arr.data.indices[ti][1] + 1 : view_offset = 0 + ts_idxs = ts_idxs .- view_offset return arr.data[idxs1..., ts_idxs, idxs2...] end @@ -646,18 +559,19 @@ function Base.setindex!(arr::TimestepArray{TS, T, N, ti}, vals, idxs::Union{Arra setindex!(arr.data, vals, idxs1..., ts_idxs, idxs2...) end -function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T_data, N, ti}, vals, idxs::Union{Array{TimestepValue{T_times},1}, AnyIndex}...) where {T_data, N, ti, FIRST, STEP, T_times} +function Base.setindex!(arr::TimestepArray{FixedTimestep{FIRST, STEP, LAST}, T_data, N, ti}, vals, idxs::Union{Array{TimestepValue{T_time},1}, AnyIndex}...) where {T_data, N, ti, FIRST, STEP, LAST, T_time} idxs1, ts_array, idxs2 = split_indices(idxs, ti) - LAST = FIRST + ((length(arr.data)-1) * STEP) ts_idxs = _get_ts_indices(ts_array, [FIRST:STEP:LAST...]) - ts_idxs = ts_idxs .- arr.data.indices[ti][1] + 1 + arr.data isa SubArray ? view_offset = arr.data.indices[ti][1] + 1 : view_offset = 0 + ts_idxs = ts_idxs .- view_offset setindex!(arr.data, vals, idxs1..., ts_idxs, idxs2...) end -function Base.setindex!(arr::TimestepArray{VariableTimestep{TIMES}, T_data, N, ti}, vals, idxs::Union{Array{TimestepValue{T_times},1}, AnyIndex}...) where {T_data, N, ti, TIMES, T_times} +function Base.setindex!(arr::TimestepArray{VariableTimestep{TIMES}, T_data, N, ti}, vals, idxs::Union{Array{TimestepValue{T_time},1}, AnyIndex}...) where {T_data, N, ti, TIMES, T_time} idxs1, ts_array, idxs2 = split_indices(idxs, ti) ts_idxs = _get_ts_indices(ts_array, TIMES) - ts_idxs = ts_idxs .- arr.data.indices[ti][1] + 1 + arr.data isa SubArray ? view_offset = arr.data.indices[ti][1] + 1 : view_offset = 0 + ts_idxs = ts_idxs .- view_offset setindex!(arr.data, vals, idxs1..., ts_idxs, idxs2...) end @@ -666,25 +580,16 @@ end Return `true` or `false`, `true` if the TimestepArray `arr` contains the Timestep `ts`. """ -function hasvalue(arr::TimestepArray{FixedTimestep{FIRST, STEP}, T, N, ti}, ts::FixedTimestep{FIRST, STEP, LAST}) where {T, N, ti, FIRST, STEP, LAST} - return 1 <= ts.t <= size(arr, 1) +function hasvalue(arr::TimestepArray{FixedTimestep{A_FIRST, A_STEP, A_LAST}, T, N, ti}, ts::FixedTimestep{T_FIRST, T_STEP, T_LAST}) where {T, N, ti, A_FIRST, A_STEP, A_LAST, T_FIRST, T_STEP, T_LAST} + return A_FIRST <= gettime(ts) <= last_period(arr) end - """ hasvalue(arr::TimestepArray, ts::VariableTimestep) Return `true` or `false`, `true` if the TimestepArray `arr` contains the Timestep `ts`. """ -function hasvalue(arr::TimestepArray{VariableTimestep{TIMES}, T, N, ti}, ts::VariableTimestep{TIMES}) where {T, N, ti, TIMES} - return 1 <= ts.t <= size(arr, 1) -end - -function hasvalue(arr::TimestepArray{FixedTimestep{D_FIRST, STEP}, T, N, ti}, ts::FixedTimestep{T_FIRST, STEP, LAST}) where {T, N, ti, D_FIRST, T_FIRST, STEP, LAST} - return D_FIRST <= gettime(ts) <= last_period(arr) -end - -function hasvalue(arr::TimestepArray{VariableTimestep{D_FIRST}, T, N, ti}, ts::VariableTimestep{T_FIRST}) where {T, N, ti, T_FIRST, D_FIRST} - return D_FIRST[1] <= gettime(ts) <= last_period(arr) +function hasvalue(arr::TimestepArray{VariableTimestep{A_TIMES}, T, N, ti}, ts::VariableTimestep{T_TIMES}) where {T, N, ti, A_TIMES, T_TIMES} + return A_TIMES[1] <= gettime(ts) <= last_period(arr) end """ @@ -693,21 +598,21 @@ end Return `true` or `false`, `true` if the TimestepArray `arr` contains the Timestep `ts` within indices `idxs`. Used when Array and Timestep have different FIRST, validating all dimensions. """ -function hasvalue(arr::TimestepArray{FixedTimestep{D_FIRST, STEP}, T, N, ti}, - ts::FixedTimestep{T_FIRST, STEP, LAST}, - idxs::Int...) where {T, N, ti, D_FIRST, T_FIRST, STEP, LAST} - return D_FIRST <= gettime(ts) <= last_period(arr) && all([1 <= idx <= size(arr, i) for (i, idx) in enumerate(idxs)]) +function hasvalue(arr::TimestepArray{FixedTimestep{A_FIRST, A_STEP, A_LAST}, T, N, ti}, + ts::FixedTimestep{T_FIRST, T_STEP, T_LAST}, + idxs::Int...) where {T, N, ti, A_FIRST, A_STEP, A_LAST, T_FIRST, T_STEP, T_LAST} + return A_FIRST <= gettime(ts) <= last_period(arr) && all([1 <= idx <= size(arr, i) for (i, idx) in enumerate(idxs)]) end """ hasvalue(arr::TimestepArray, ts::VariableTimestep, idxs::Int...) Return `true` or `false`, `true` if the TimestepArray `arr` contains the Timestep `ts` within -indices `idxs`. Used when Array and Timestep different TIMES, validating all dimensions. +indices `idxs`. Used when Array and Timestep have different TIMES, validating all dimensions. """ -function hasvalue(arr::TimestepArray{VariableTimestep{D_FIRST}, T, N, ti}, - ts::VariableTimestep{T_FIRST}, - idxs::Int...) where {T, N, ti, D_FIRST, T_FIRST} +function hasvalue(arr::TimestepArray{VariableTimestep{A_TIMES}, T, N, ti}, + ts::VariableTimestep{T_TIMES}, + idxs::Int...) where {T, N, ti, A_TIMES, T_TIMES} - return D_FIRST[1] <= gettime(ts) <= last_period(arr) && all([1 <= idx <= size(arr, i) for (i, idx) in enumerate(idxs)]) + return A_TIMES[1] <= gettime(ts) <= last_period(arr) && all([1 <= idx <= size(arr, i) for (i, idx) in enumerate(idxs)]) end diff --git a/src/core/types/time.jl b/src/core/types/time.jl index 7bdde8183..4bd781abd 100644 --- a/src/core/types/time.jl +++ b/src/core/types/time.jl @@ -60,8 +60,8 @@ mutable struct Clock{T <: AbstractTimestep} <: MimiStruct end end -# TDESIGN DISCUSSION: add this for performance? -# mutable struct TimestepArray{T_TS <: AbstractTimestep, T, N, ti, S<:AbstractArray{T,N}} <: MimiStruct +# DESIGN DISCUSSION: add this for performance? +# struct TimestepArray{T_TS <: AbstractTimestep, T, N, ti, S<:AbstractArray{T,N}} <: MimiStruct # data::S mutable struct TimestepArray{T_TS <: AbstractTimestep, T, N, ti} <: MimiStruct data::Union{Array{T, N}, SubArray} diff --git a/test/test_parametertypes.jl b/test/test_parametertypes.jl index 101e6f78d..eb97c0585 100644 --- a/test/test_parametertypes.jl +++ b/test/test_parametertypes.jl @@ -84,8 +84,8 @@ extpars = external_params(m.mi.md) @test isa(extpars[:e], ArrayModelParameter) @test isa(extpars[:f], ScalarModelParameter) # note that :f is stored as a scalar parameter even though its values are an array -@test typeof(extpars[:a].values) == TimestepMatrix{FixedTimestep{2000, 1}, arrtype, 1} -@test typeof(extpars[:b].values) == TimestepVector{FixedTimestep{2000, 1}, arrtype} +@test typeof(extpars[:a].values) == TimestepMatrix{FixedTimestep{2000, 1, 2100}, arrtype, 1} +@test typeof(extpars[:b].values) == TimestepVector{FixedTimestep{2000, 1, 2100}, arrtype} @test typeof(extpars[:c].values) == Array{arrtype, 1} @test typeof(extpars[:d].value) == numtype @@ -111,7 +111,7 @@ update_param!(m, :e, [4,5,6,7]) @test length(extpars) == 9 # The old dictionary has the default values that were added during build, so it has more entries @test length(new_extpars) == 6 -@test typeof(new_extpars[:a].values) == TimestepMatrix{FixedTimestep{2000, 1}, arrtype, 1} +@test typeof(new_extpars[:a].values) == TimestepMatrix{FixedTimestep{2000, 1, 2100}, arrtype, 1} @test typeof(new_extpars[:d].value) == numtype @test typeof(new_extpars[:e].values) == Array{arrtype, 1} @@ -141,7 +141,7 @@ set_param!(m, :MyComp2, :x, [1, 2, 3, 4, 5]) set_dimension!(m, :time, 2001:2003) update_param!(m, :x, [2, 3, 4]) x = external_param(m.md, :x) -@test x.values isa Mimi.TimestepArray{Mimi.FixedTimestep{2001, 1, LAST} where LAST, Union{Missing,Float64}, 1} +@test x.values isa Mimi.TimestepArray{Mimi.FixedTimestep{2001, 1, 2003}, Union{Missing,Float64}, 1} @test x.values.data == [2., 3., 4.] run(m) @test m[:MyComp2, :y][1] == 2 # 2001 @@ -196,7 +196,7 @@ set_dimension!(m, :time, 1999:2003) # length 5 update_param!(m, :x, [2, 3, 4, 5, 6]) x = external_param(m.md, :x) -@test x.values isa Mimi.TimestepArray{Mimi.FixedTimestep{1999, 1, LAST} where LAST, Union{Missing,Float64}, 1} +@test x.values isa Mimi.TimestepArray{Mimi.FixedTimestep{1999, 1, 2003}, Union{Missing, Float64}, 1, 1} @test x.values.data == [2., 3., 4., 5., 6.] run(m) @@ -230,7 +230,7 @@ set_dimension!(m, :time, 2005:2007) update_params!(m, Dict(:x=>[3,4,5], :y=>[10,20], :z=>0)) # Won't error when updating from a dictionary -@test external_param(m.md, :x).values isa Mimi.TimestepArray{Mimi.FixedTimestep{2005,1},Union{Missing,Float64},1} +@test external_param(m.md, :x).values isa Mimi.TimestepArray{Mimi.FixedTimestep{2005,1, 2007},Union{Missing,Float64},1} @test external_param(m.md, :x).values.data == [3.,4.,5.] @test external_param(m.md, :y).values == [10.,20.] @test external_param(m.md, :z).value == 0 diff --git a/test/test_show.jl b/test/test_show.jl index 33fd09769..30d4fb075 100644 --- a/test/test_show.jl +++ b/test/test_show.jl @@ -81,8 +81,8 @@ Model param_name: :x external_param: :x external_params: Dict{Symbol,ModelParameter} - x => ArrayModelParameter{TimestepArray{FixedTimestep{2000,1,LAST} where LAST,Float64,1}} - values: TimestepArray{FixedTimestep{2000,1,LAST} where LAST,Float64,1} + x => ArrayModelParameter{TimestepArray{FixedTimestep{2000,1,2005},Float64,1}} + values: TimestepArray{FixedTimestep{2000,1,2005},Float64,1} 1: 0.0 2: 0.0 3: 0.0 diff --git a/test/test_timesteparrays.jl b/test/test_timesteparrays.jl index d078c6909..01ee254fe 100644 --- a/test/test_timesteparrays.jl +++ b/test/test_timesteparrays.jl @@ -42,7 +42,7 @@ idx4 = TimestepIndex(4) #1a. test constructor, lastindex, and length (with both # matching years and mismatched years) -x = TimestepVector{FixedTimestep{2000, 1}, Int}([9, 10, 11, 12]) +x = TimestepVector{FixedTimestep{2000, 1, 2003}, Int}([9, 10, 11, 12]) @test length(x) == 4 @test lastindex(x) == TimestepIndex(4) @@ -80,21 +80,22 @@ reset_time_val(x, time_dim_val) # AbstractTimestep Indexing t = FixedTimestep{2001, 1, 3000}(1) - @test hasvalue(x, t) @test !hasvalue(x, FixedTimestep{2000, 1, 2012}(10)) -@test x[t] == 10 + +t = FixedTimestep{2000, 1, 3000}(1) +@test x[t] == 9 # x t2 = next_timestep(t) -@test x[t2] == time_dim_val[3] -x[t2] = temp_dim_val[3] -@test x[t2] == temp_dim_val[3] +@test x[t2] == time_dim_val[2] +x[t2] = temp_dim_val[2] +@test x[t2] == temp_dim_val[2] reset_time_val(x, time_dim_val) -t3 = FixedTimestep{2000, 1, 2003}(1) -@test x[t3] == time_dim_val[1] -x[t3] = temp_dim_val[1] -@test x[t3] == temp_dim_val[1] +t3 = next_timestep(t2) +@test x[t3] == time_dim_val[3] +x[t3] = temp_dim_val[3] +@test x[t3] == temp_dim_val[3] reset_time_val(x, time_dim_val) # Deprecated int indexing now errors @@ -148,12 +149,12 @@ t = VariableTimestep{y2}() @test hasvalue(x, t) @test !hasvalue(x, VariableTimestep{years}(time_dim_val[2])) -@test x[t] == time_dim_val[2] +@test x[t] == time_dim_val[1] t2 = next_timestep(t) -@test x[t2] == time_dim_val[3] -x[t2] = temp_dim_val[3] -@test x[t2] == temp_dim_val[3] +@test x[t2] == time_dim_val[2] +x[t2] = temp_dim_val[2] +@test x[t2] == temp_dim_val[2] reset_time_val(x, time_dim_val) t3 = VariableTimestep{years}() @@ -176,8 +177,8 @@ for ti = 1:2 #3a. test constructor (with both matching years # and mismatched years) - y = TimestepMatrix{FixedTimestep{2000, 1}, Int, ti}(collect(reshape(1:8, 4, 2))) - z = TimestepMatrix{FixedTimestep{2000, 2}, Int, ti}(collect(reshape(1:8, 4, 2))) + y = TimestepMatrix{FixedTimestep{2000, 1, 2003}, Int, ti}(collect(reshape(1:8, 4, 2))) + z = TimestepMatrix{FixedTimestep{2000, 2, 2003}, Int, ti}(collect(reshape(1:8, 4, 2))) time_dim_val = collect(reshape(1:8, 4, 2)) temp_dim_val = collect(reshape(100:107, 4, 2)) @@ -239,8 +240,8 @@ for ti = 1:2 @test y[:, TimestepValue(2001)] == time_dim_val[:,2] @test y[1, TimestepValue(2000; offset = 1)] == time_dim_val[1,2] @test y[1, TimestepValue(2000) + 1] == time_dim_val[1,2] - @test_throws ErrorException y[1, TimestepValue(2003)] - @test_throws ErrorException y[1, TimestepValue(2002)+1] + @test_throws BoundsError y[1, TimestepValue(2003)] + @test_throws BoundsError y[1, TimestepValue(2002)+1] y[1, TimestepValue(2000)] = temp_dim_val[1] @test y[1, TimestepValue(2000)] == temp_dim_val[1] @@ -248,10 +249,10 @@ for ti = 1:2 end # AbstractTimestep Indexing - t = FixedTimestep{2001, 1, 3000}(1) + t = FixedTimestep{2000, 1, 3000}(1) @test hasvalue(y, t, 1) @test !hasvalue(y, FixedTimestep{2000, 1, 3000}(10), 1) - + t = next_timestep(t) if ti == 1 @test y[t,1] == time_dim_val[2,1] @test y[t,2] == time_dim_val[2,2] @@ -271,7 +272,7 @@ for ti = 1:2 reset_time_val(y, time_dim_val) #3c. interval wider than 1 using z from above - t = FixedTimestep{1980, 2, 3000}(11) + t = FixedTimestep{1980, 2, 3000}(1) @test z[t,1] == time_dim_val[1,1] @test z[t,2] == time_dim_val[1,2] @@ -292,7 +293,7 @@ for ti = 1:2 reset_time_val(y, time_dim_val) #3c. interval wider than 1 using z from above - t = FixedTimestep{1980, 2, 3000}(11) + t = FixedTimestep{1980, 2, 3000}(1) @test z[1, t] == time_dim_val[1,1] @test z[2, t] == time_dim_val[2,1] @@ -402,14 +403,16 @@ for ti = 1:2 if ti == 1 @test hasvalue(y, t, time_dim_val[1]) @test !hasvalue(y, VariableTimestep{years}(10)) - @test y[t,1] == time_dim_val[2,1] - @test y[t,2] == time_dim_val[2,2] t2 = next_timestep(t) - @test y[t2,1] == time_dim_val[3,1] - @test y[t2,2] == time_dim_val[3,2] - y[t2, 1] = temp_dim_val[3,1] - @test y[t2, 1] == temp_dim_val[3,1] + @test y[t2,1] == time_dim_val[2,1] + @test y[t2,2] == time_dim_val[2,2] + + t3 = next_timestep(t2) + @test y[t3,1] == time_dim_val[3,1] + @test y[t3,2] == time_dim_val[3,2] + y[t3, 1] = temp_dim_val[3,1] + @test y[t3, 1] == temp_dim_val[3,1] reset_time_val(y, time_dim_val) t3 = VariableTimestep{years}() @@ -422,8 +425,10 @@ for ti = 1:2 else @test hasvalue(y, t, time_dim_val[1]) @test !hasvalue(y, VariableTimestep{years}(10)) - @test y[1,t] == time_dim_val[1,2] - @test y[2,t] == time_dim_val[2,2] + + t2 = next_timestep(t) + @test y[1,t2] == time_dim_val[1,2] + @test y[2,t2] == time_dim_val[2,2] t3 = VariableTimestep{years}() @test y[1, t3] == time_dim_val[1,1] @@ -446,7 +451,7 @@ end for ti = 1:2 years = Tuple([2000:5:2005; 2015:10:2025]) - arr_fixed = TimestepArray{FixedTimestep{2000, 5}, Int, 3, ti}(collect(reshape(1:64, 4, 4, 4))) + arr_fixed = TimestepArray{FixedTimestep{2000, 5, 2020}, Int, 3, ti}(collect(reshape(1:64, 4, 4, 4))) arr_variable = TimestepArray{VariableTimestep{years}, Int, 3, ti}(collect(reshape(1:64, 4, 4, 4))) time_dim_val = collect(reshape(1:64, 4, 4, 4)) @@ -591,8 +596,8 @@ time_dim_val = collect(reshape(1:64, 4, 4, 4)) x_years = Tuple(2000:5:2015) #fixed y_years = Tuple([2000:5:2005; 2015:10:2025]) #variable -x_vec = TimestepVector{FixedTimestep{2000, 5}, Int}(time_dim_val[:,1,1]) -x_mat = TimestepMatrix{FixedTimestep{2000, 5}, Int, 1}(time_dim_val[:,:,1]) +x_vec = TimestepVector{FixedTimestep{2000, 5, 2015}, Int}(time_dim_val[:,1,1]) +x_mat = TimestepMatrix{FixedTimestep{2000, 5, 2015}, Int, 1}(time_dim_val[:,:,1]) y_vec = TimestepVector{VariableTimestep{y_years}, Int}(time_dim_val[:,2,2]) y_mat = TimestepMatrix{VariableTimestep{y_years}, Int, 1}(time_dim_val[:,:,2]) @@ -717,11 +722,8 @@ run(m) # 8. Check broadcast assignment to underlying array #------------------------------------------------------------------------------ -x_arr = zeros(10) -y_arr = collect(reshape(zeros(8), 4, 2)) - -x = Mimi.TimestepVector{Mimi.FixedTimestep{2005,10}, Float64}(x_arr) -y = TimestepMatrix{FixedTimestep{2000, 1}, Float64, 1}(y_arr) +x = Mimi.TimestepVector{Mimi.FixedTimestep{2005,10}, Float64}(zeros(10)) +y = TimestepMatrix{FixedTimestep{2000, 1}, Float64, 1}(collect(reshape(zeros(8), 4, 2))) # colon and ints x[:] .= 10 @@ -732,14 +734,14 @@ y[:] .= 10 y[:,1] .= 20 @test all(y.data[:,1] .== 20) -reset_time_val(x, x_arr) -reset_time_val(y, y_arr) +reset_time_val(x, zeros(10)) +reset_time_val(y, collect(reshape(zeros(8), 4, 2))) # TimestepIndex y[TimestepIndex(2),:] .= 10 @test all(y.data[2,:] .== 10) -reset_time_val(x, x_arr) -reset_time_val(y, y_arr) +reset_time_val(x, zeros(10)) +reset_time_val(y, collect(reshape(zeros(8), 4, 2))) end #module From 2e63e0c2bfbb40a19116267b0c89f3f718b6878c Mon Sep 17 00:00:00 2001 From: lrennels Date: Sat, 20 Feb 2021 15:35:18 -0800 Subject: [PATCH 24/32] Revert tests back to proper array output expectation --- src/components/connector.jl | 2 +- src/core/build.jl | 2 +- src/core/connections.jl | 10 +--------- test/test_firstlast.jl | 35 +++++++++++++++++------------------ test/test_timesteps.jl | 14 ++++++-------- 5 files changed, 26 insertions(+), 37 deletions(-) diff --git a/src/components/connector.jl b/src/components/connector.jl index 33cb78870..da19e766c 100644 --- a/src/components/connector.jl +++ b/src/components/connector.jl @@ -62,7 +62,7 @@ end # elseif hasvalue(p.input2, t) # v.output[t, colons...] = p.input2[t, colons...] # else -# error("Neither of the input to ConnectorComp have data for the current timestep: $(gettime(t)).") +# error("Neither of the inputs to ConnectorComp have data for the current timestep: $(gettime(t)).") # end # end # end diff --git a/src/core/build.jl b/src/core/build.jl index c1a508c3b..efe5cdfb2 100644 --- a/src/core/build.jl +++ b/src/core/build.jl @@ -247,7 +247,7 @@ end function _instantiate_params(comp_def::ComponentDef, par_dict::Dict{Tuple{ComponentPath, Symbol}, Any}) # @info "Instantiating params for $(comp_def.comp_path)" comp_path = comp_def.comp_path - names = parameter_names(comp_def) + names = parameter_names(comp_def) vals = Any[par_dict[(comp_path, name)] for name in names] _substitute_views!(vals, comp_def) types = DataType[typeof(val) for val in vals] diff --git a/src/core/connections.jl b/src/core/connections.jl index a17c8d80e..7d86404aa 100644 --- a/src/core/connections.jl +++ b/src/core/connections.jl @@ -69,15 +69,7 @@ function _check_labels(obj::AbstractCompositeComponentDef, for (i, dim) in enumerate(comp_dims) if isa(dim, Symbol) param_length = size(ext_param.values)[i] - # we are now requiring time lengths to match the model as well - # if dim == :time - # t = dimension(obj, :time) - # first = find_first_period(comp_def) - # last = find_last_period(comp_def) - # comp_length = t[last] - t[first] + 1 - # else - comp_length = dim_count(obj, dim) - # end + comp_length = dim_count(obj, dim) if param_length != comp_length error("Mismatched data size for a parameter connection: dimension :$dim in $(comp_def.comp_id) has $comp_length elements; external parameter :$param_name has $param_length elements.") end diff --git a/test/test_firstlast.jl b/test/test_firstlast.jl index 5ea74a152..7715e50bb 100644 --- a/test/test_firstlast.jl +++ b/test/test_firstlast.jl @@ -73,9 +73,9 @@ run(m) # test that there are missing values in :emissions variables outside of the component's # run period, and no missing values in the :grosseconomy variables -@test ismissing(m[:emissions, :E].parent[1]) -@test ismissing(m[:emissions, :E].parent[20]) -@test sum(ismissing.(m[:emissions, :E])) == 0 +@test ismissing(m[:emissions, :E][1]) +@test ismissing(m[:emissions, :E][20]) +@test sum(ismissing.(m[:emissions, :E][2:19])) == 0 @test sum(ismissing.(m[:grosseconomy, :l])) == 0 # change the model dimension (widen it) @@ -99,9 +99,9 @@ run(m) # test that there are missing values in :emissions variables outside of the component's # run period, and no missing values in the :grosseconomy variables -@test ismissing(m[:emissions, :E].parent[1]) -@test sum(ismissing.(m[:emissions, :E].parent[20:21])) == 2 -@test sum(ismissing.(m[:emissions, :E])) == 0 +@test ismissing(m[:emissions, :E][1]) + @test sum(ismissing.(m[:emissions, :E][20:21])) == 2 + @test sum(ismissing.(m[:emissions, :E][2:19])) == 0 @test sum(ismissing.(m[:grosseconomy, :l])) == 0 # @@ -128,8 +128,8 @@ connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) run(m) # neither component should have a value for the first timestep -@test ismissing(m[:emissions, :E].parent[1]) -@test ismissing(m[:grosseconomy, :YGROSS].parent[1]) +@test ismissing(m[:emissions, :E][1]) +@test ismissing(m[:grosseconomy, :YGROSS][1]) # Test bounds - both end early @@ -154,8 +154,8 @@ connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) run(m) -@test ismissing(m[:emissions, :E].parent[20]) -@test ismissing(m[:grosseconomy, :YGROSS].parent[20]) +@test ismissing(m[:emissions, :E][20]) +@test ismissing(m[:grosseconomy, :YGROSS][20]) # # Test bounds - components starting or ending before/after the model @@ -207,9 +207,9 @@ set_dimension!(m2, :time, collect(1:10)) add_comp!(m2, MyComp, first = 2, last = 9) run(m2) -@test ismissing(m[:MyComp, :a].parent[1]) -@test ismissing(m[:MyComp, :a].parent[end]) -@test (m[:MyComp, :a]) == (m2[:MyComp, :a]) == [-999., 2., 3., 4., 5., 6., 7., 999.] +@test ismissing(m[:MyComp, :a][1]) +@test ismissing(m[:MyComp, :a][end]) +@test (m[:MyComp, :a])[2:9] == (m2[:MyComp, :a])[2:9] == [-999., 2., 3., 4., 5., 6., 7., 999.] # # TimestepIndex and TimestepValue @@ -231,7 +231,7 @@ add_comp!(m, MyComp, first = 5, last = 10) run(m) for i in collect(1:15) - @test m[:MyComp, :a].parent[i] === m[:MyComp, :b].parent[i] + @test m[:MyComp, :a][i] === m[:MyComp, :b][i] end # TimestepValue: Test Equality - should match up with the time index @@ -250,7 +250,7 @@ add_comp!(m, MyComp, first = 5, last = 10) run(m) for i in collect(1:15) - @test m[:MyComp, :a].parent[i] === m[:MyComp, :b].parent[i] + @test m[:MyComp, :a][i] === m[:MyComp, :b][i] end # TimestepIndex: Test that Get and Set Index are Relative to Component, not Model @@ -286,8 +286,7 @@ for year in collect(1995:1999) add_comp!(m, MyComp, first = year) run(m) idx = year - 1995 + 1 - @test m[:MyComp, :a].parent[idx] == 1.0 - @test m[:MyComp, :a][1] == 1.0 + @test m[:MyComp, :a][idx] == 1.0 end # TimestepValue: Test that Get and Set Index are Relative to Component, not Model @@ -307,7 +306,7 @@ for year in collect(1995:1999) set_dimension!(m, :time, collect(1995:2000)) add_comp!(m, MyComp, first = year) run(m) - @test m[:MyComp, :a].parent[5] == 1.0 + @test m[:MyComp, :a][5] == 1.0 end end #module diff --git a/test/test_timesteps.jl b/test/test_timesteps.jl index 987cfc3a5..7e225ddfd 100644 --- a/test/test_timesteps.jl +++ b/test/test_timesteps.jl @@ -150,19 +150,17 @@ set_param!(m, :Bar, :inputB, collect(1:length(years))) run(m) -@test length(m[:Foo, :output].parent) == length(years) -@test length(m[:Bar, :output].parent) == length(years) - -@test length(m[:Foo, :output]) == length(first_foo:years[end]) +@test length(m[:Foo, :output]) == length(years) @test length(m[:Bar, :output]) == length(years) yr_dim = Mimi.Dimension(years) idxs = yr_dim[first_foo]:yr_dim[years[end]] foo_output = m[:Foo, :output] -for i in 1:5 - @test foo_output[i] == 5+i -end +offset = first_foo - years[1] + for i in idxs + @test foo_output[i] == 5+(i-offset) # incorporate offset into i now because we set ts.t to match component not model + end for i in 1:5 @test m[:Bar, :output][i] == i @@ -224,7 +222,7 @@ connect_param!(m2, :Foo2, :inputF, :Bar, :output) run(m2) -foo_output2 = m2[:Foo2, :output] +foo_output2 = m2[:Foo2, :output][yr_dim[first_foo]:yr_dim[years[end]]] for i in 1:6 @test foo_output2[i] == (i+5)^2 end From 2706d205a9a499e3e1073680ac39e7fd79c69bde Mon Sep 17 00:00:00 2001 From: lrennels Date: Mon, 22 Feb 2021 15:45:02 -0800 Subject: [PATCH 25/32] Fix connector problem --- src/core/build.jl | 4 +++- src/core/instances.jl | 6 +++++- src/core/time_arrays.jl | 6 +----- test/test_firstlast.jl | 6 ++++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/core/build.jl b/src/core/build.jl index efe5cdfb2..b44bb9465 100644 --- a/src/core/build.jl +++ b/src/core/build.jl @@ -18,7 +18,9 @@ end function _get_view(val::TimestepArray{T_TS, T, N, ti}, first_idx, last_idx) where {T_TS, T, N, ti} idxs = Array{Any}(fill(:, N)) idxs[ti] = first_idx:last_idx - return view(val.data, idxs...) + # if we are making a connection, the val.data may already be a view, in which case + # we need to return to the parent of that view before taking our new view + return val.data isa SubArray ? view(val.data.parent, idxs...) : view(val.data, idxs...) end # Return the datatype to use for instance variables/parameters diff --git a/src/core/instances.jl b/src/core/instances.jl index 2289deeef..387d47675 100644 --- a/src/core/instances.jl +++ b/src/core/instances.jl @@ -206,7 +206,11 @@ function _get_datum(ci::LeafComponentInstance, datum_name::Symbol) value = getproperty(which, datum_name) - return value isa TimestepArray ? value.data : value + if value isa TimestepArray + return value.data isa SubArray ? parent(value.data) : value.data + else + return value + end end function Base.getindex(mi::ModelInstance, key::AbstractString, datum::Symbol) diff --git a/src/core/time_arrays.jl b/src/core/time_arrays.jl index 3dd0486b7..a93084c41 100644 --- a/src/core/time_arrays.jl +++ b/src/core/time_arrays.jl @@ -104,7 +104,7 @@ function _get_time_value_position(times::Union{Tuple, Array}, ts::TimestepValue{ return t_offset end -# Helper function to get the array of indices from an Array{TimestepIndex,1} +# Helper function to get the array of indices from an Array{TimestepIndex,1} or Array{TimestepValue, 1} function _get_ts_indices(ts_array::Array{TimestepIndex, 1}) return [ts.index for ts in ts_array] end @@ -113,10 +113,6 @@ function _get_ts_indices(ts_array::Array{TimestepValue{T}, 1}, times::Union{Tupl return [_get_time_value_position(times, ts) for ts in ts_array] end -function _get_ts_indices(arr::Array{Colon, N}) where N - return [arr...] -end - # Base.firstindex and Base.lastindex function Base.firstindex(arr::TimestepArray{T_TS, T, N, ti}) where {T_TS, T, N, ti} if ti == 1 diff --git a/test/test_firstlast.jl b/test/test_firstlast.jl index 7715e50bb..1b07f942f 100644 --- a/test/test_firstlast.jl +++ b/test/test_firstlast.jl @@ -123,7 +123,7 @@ set_param!(m, :grosseconomy, :share, 0.3) # Set parameters for the emissions component set_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20]) -connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) +connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS) run(m) @@ -267,7 +267,9 @@ m = Model() set_dimension!(m, :time, collect(1:15)) add_comp!(m, MyComp, first = 5, last = 10) run(m) -@test m[:MyComp, :a] == m[:MyComp, :b] +for (i, element) in enumerate(m[:MyComp, :a]) + @test element === m[:MyComp, :b][i] +end @defcomp MyComp begin a = Variable(index=[time]) From edb9a7100482fd09ad675a106b5dfe5c36496316 Mon Sep 17 00:00:00 2001 From: lrennels Date: Mon, 22 Feb 2021 15:58:08 -0800 Subject: [PATCH 26/32] Fix bug in timesteparray test --- test/test_timesteparrays.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_timesteparrays.jl b/test/test_timesteparrays.jl index 01ee254fe..2f1672f07 100644 --- a/test/test_timesteparrays.jl +++ b/test/test_timesteparrays.jl @@ -722,8 +722,8 @@ run(m) # 8. Check broadcast assignment to underlying array #------------------------------------------------------------------------------ -x = Mimi.TimestepVector{Mimi.FixedTimestep{2005,10}, Float64}(zeros(10)) -y = TimestepMatrix{FixedTimestep{2000, 1}, Float64, 1}(collect(reshape(zeros(8), 4, 2))) +x = Mimi.TimestepVector{Mimi.FixedTimestep{2005,10,2095}, Float64}(zeros(10)) +y = TimestepMatrix{FixedTimestep{2000, 1, 2003}, Float64, 1}(collect(reshape(zeros(8), 4, 2))) # colon and ints x[:] .= 10 From cd91c6b8f9e8c08870a85921f2808a8c50b9da58 Mon Sep 17 00:00:00 2001 From: lrennels Date: Mon, 22 Feb 2021 22:24:20 -0800 Subject: [PATCH 27/32] Fix _update_param! bug with explicit typing --- src/core/connections.jl | 8 ++++++-- test/test_parametertypes.jl | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/core/connections.jl b/src/core/connections.jl index 7d86404aa..028e24e5f 100644 --- a/src/core/connections.jl +++ b/src/core/connections.jl @@ -545,14 +545,18 @@ function _update_array_param!(obj::AbstractCompositeComponentDef, name, value) # check if updating timestep labels is necessary if param.values isa TimestepArray time_label_change = time_labels(param.values) != dim_keys(obj, :time) + N = ndims(value) if time_label_change T = eltype(value) - N = length(size(value)) ti = get_time_index_position(param) new_timestep_array = get_timestep_array(obj, T, N, ti, value) set_external_param!(obj, name, ArrayModelParameter(new_timestep_array, dim_names(param))) else - param.values.data = value + # DESIGN DISCUSSION - this became necessary when the Array type for + # TimestepArray widened to include SubArray, can check if it's necessary + # using test_parametertypes.jl around line 141 + T = eltype(param.values) + param.values.data = Array{T,N}(value) end else param.values = value diff --git a/test/test_parametertypes.jl b/test/test_parametertypes.jl index eb97c0585..c785c161e 100644 --- a/test/test_parametertypes.jl +++ b/test/test_parametertypes.jl @@ -138,6 +138,15 @@ set_param!(m, :MyComp2, :x, [1, 2, 3, 4, 5]) @test_throws ErrorException set_dimension!(m, :time, 2002:2004) # model starts after component @test_throws ErrorException set_dimension!(m, :time, 2000:2002) # model ends before component + +# these tests specifically look for the case of updating a TimestepArray's data +# without time labels, thus simply calling param.values.data = new_data ... the addition +# of SubArray to data's type options made it necessary to convert types explicitly +# which may be causing a performance hit +update_param!(m, :x, [2.,3.,4.,5.,6.]) +update_param!(m, :x, zeros(5)) +update_param!(m, :x, [1,2,3,4,5]) + set_dimension!(m, :time, 2001:2003) update_param!(m, :x, [2, 3, 4]) x = external_param(m.md, :x) From af58cb52a0a8863b4b82ac247d55b50c478dcf9f Mon Sep 17 00:00:00 2001 From: lrennels Date: Fri, 26 Feb 2021 11:55:20 -0800 Subject: [PATCH 28/32] Clean up --- docs/src/howto/howto_1.md | 2 +- src/core/instances.jl | 2 +- src/core/model.jl | 2 +- src/core/types/time.jl | 6 +++++- test/runtests.jl | 2 +- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/src/howto/howto_1.md b/docs/src/howto/howto_1.md index 21797c166..d9617ae0b 100644 --- a/docs/src/howto/howto_1.md +++ b/docs/src/howto/howto_1.md @@ -73,7 +73,7 @@ add_comp!(m, ComponentA, :GDP) The first argument to `add_comp!` is the model, the second is the name of the ComponentId defined by `@defcomp`. If an optional third symbol is provided (as in the second line above), this will be used as the name of the component in this model. This allows you to add multiple versions of the same component to a model, with different names. -The `add_comp` function has two more optional keyword arguments, `first` and `last`, which can be used to indicate a fixed start and/or end time (year in this case) that the compnonent should run for (within the bounds of the model's time dimension). For example, the following sould indicate that `ComponentA` should only run from 1900 to 2000. +The `add_comp` function has two more optional keyword arguments, `first` and `last`, which can be used to indicate a fixed start and/or end time (year in this case) that the compnonent should run for (within the bounds of the model's time dimension). For example, the following indicates that `ComponentA` should only run from 1900 to 2000. ```julia add_comp!(m, ComponentA; first = 1900, last = 2000) diff --git a/src/core/instances.jl b/src/core/instances.jl index 387d47675..ceb233e00 100644 --- a/src/core/instances.jl +++ b/src/core/instances.jl @@ -286,7 +286,7 @@ function get_shifted_ts(ci, ts::FixedTimestep{FIRST, STEP, LAST}) where {FIRST, if ci.first == FIRST && ci.last == LAST return ts else - # shift the timestep over by (ci.first = FIRST)/STEP + # shift the timestep over by (ci.first - FIRST)/STEP return FixedTimestep{ci.first,STEP,ci.last}(ts.t - Int((ci.first - FIRST)/STEP)) end end diff --git a/src/core/model.jl b/src/core/model.jl index ca14dbb81..a07594fad 100644 --- a/src/core/model.jl +++ b/src/core/model.jl @@ -114,7 +114,7 @@ For each (k, v) in the provided `parameters` dictionary, `update_param!`` is called to update the external parameter by name k to value v. Each key k must be a symbol or convert to a symbol matching the name of an external parameter t hat already exists in the model definition. The update_timesteps keyword argument -is deprecated, we keep it here just to provide warnings. +is deprecated, but temporarily remains as a dummy argument to allow warning detection. """ @delegate update_params!(m::Model, parameters::Dict) => md diff --git a/src/core/types/time.jl b/src/core/types/time.jl index 4bd781abd..be92a9339 100644 --- a/src/core/types/time.jl +++ b/src/core/types/time.jl @@ -60,7 +60,11 @@ mutable struct Clock{T <: AbstractTimestep} <: MimiStruct end end -# DESIGN DISCUSSION: add this for performance? +# DESIGN DISCUSSION: how should we parameterize this for performance and clarity? +# One idea is below, and note that if we add parameterization we need to change +# all instances of the constructor in the codebase, and make sure we are able to +# construct it in all cases. +# # struct TimestepArray{T_TS <: AbstractTimestep, T, N, ti, S<:AbstractArray{T,N}} <: MimiStruct # data::S mutable struct TimestepArray{T_TS <: AbstractTimestep, T, N, ti} <: MimiStruct diff --git a/test/runtests.jl b/test/runtests.jl index 557b89007..00f10e7d0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,7 +13,7 @@ using Documenter Electron.prep_test_env() @testset "Mimi" begin - + @info("test_main.jl") @time include("test_main.jl") From ac6eb5584361272e96cef44018edf8b72e3c56a9 Mon Sep 17 00:00:00 2001 From: lrennels Date: Mon, 1 Mar 2021 23:14:47 -0800 Subject: [PATCH 29/32] Add performance improvements --- src/core/build.jl | 19 ++++++++++++------- src/core/types/time.jl | 25 ++++++++----------------- test/test_parametertypes.jl | 6 +++--- 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/src/core/build.jl b/src/core/build.jl index b44bb9465..d29601707 100644 --- a/src/core/build.jl +++ b/src/core/build.jl @@ -6,21 +6,19 @@ function _substitute_views!(vals::Array{T, N}, comp_def) where {T, N} first_idx = findfirst(times .== comp_def.first) last_idx = findfirst(times .== comp_def.last) for (i, val) in enumerate(vals) - # DESIGN DISCUSSION should we deep copy here or can we just do in-place val.data = _get_view(val, first_idx, last_idx) if val isa TimestepArray - new_val = deepcopy(val) - new_val.data = _get_view(val, first_idx, last_idx) - vals[i] = new_val + vals[i] = _get_view(val, first_idx, last_idx) end end end -function _get_view(val::TimestepArray{T_TS, T, N, ti}, first_idx, last_idx) where {T_TS, T, N, ti} +function _get_view(val::TimestepArray{T_TS, T, N, ti, S}, first_idx, last_idx) where {T_TS, T, N, ti, S} + idxs = Array{Any}(fill(:, N)) idxs[ti] = first_idx:last_idx # if we are making a connection, the val.data may already be a view, in which case # we need to return to the parent of that view before taking our new view - return val.data isa SubArray ? view(val.data.parent, idxs...) : view(val.data, idxs...) + return TimestepArray{T_TS, T, N, ti}(val.data isa SubArray ? view(val.data.parent, idxs...) : view(val.data, idxs...)) end # Return the datatype to use for instance variables/parameters @@ -95,7 +93,14 @@ function _instantiate_component_vars(md::ModelDef, comp_def::ComponentDef) names = Symbol[nameof(def) for def in var_defs] values = Any[_instantiate_datum(md, def) for def in var_defs] _substitute_views!(values, comp_def) - types = DataType[_instance_datatype(md, def) for def in var_defs] + + # this line was replaced with the one below because calling _instance_datatype + # does not concretely type the S type parameter (the type of the TimestepArray's + # and thus DataType[typeof(val) for val in values] errored when trying to + # convert typeof(val<:TimestepArray) to a DataType + + # types = DataType[_instance_datatype(md, def) for def in var_defs] + types = DataType[typeof(val) for val in values] paths = repeat(Any[comp_def.comp_path], length(names)) return ComponentInstanceVariables(names, types, values, paths) diff --git a/src/core/types/time.jl b/src/core/types/time.jl index be92a9339..1e5473b4f 100644 --- a/src/core/types/time.jl +++ b/src/core/types/time.jl @@ -60,27 +60,18 @@ mutable struct Clock{T <: AbstractTimestep} <: MimiStruct end end -# DESIGN DISCUSSION: how should we parameterize this for performance and clarity? -# One idea is below, and note that if we add parameterization we need to change -# all instances of the constructor in the codebase, and make sure we are able to -# construct it in all cases. -# -# struct TimestepArray{T_TS <: AbstractTimestep, T, N, ti, S<:AbstractArray{T,N}} <: MimiStruct -# data::S -mutable struct TimestepArray{T_TS <: AbstractTimestep, T, N, ti} <: MimiStruct - data::Union{Array{T, N}, SubArray} +mutable struct TimestepArray{T_TS <: AbstractTimestep, T, N, ti, S<:AbstractArray{T,N}} <: MimiStruct + + data::S - function TimestepArray{T_TS, T, N, ti}(d::SubArray) where {T_TS, T, N, ti} - return new(d) + function TimestepArray{T_TS, T, N, ti}(d::S) where {T_TS, T, N, ti, S} + return new{T_TS, T, N, ti, S}(d) end - function TimestepArray{T_TS, T, N, ti}(d::Array{T, N}) where {T_TS, T, N, ti} - return new(d) - end +end - function TimestepArray{T_TS, T, N, ti}(lengths::Int...) where {T_TS, T, N, ti} - return new(Array{T, N}(undef, lengths...)) - end +function TimestepArray{T_TS, T, N, ti}(lengths::Int...) where {T_TS, T, N, ti} + return TimestepArray{T_TS, T, N, ti}(Array{T, N}(undef, lengths...)) end # Since these are the most common cases, we define methods (in time.jl) diff --git a/test/test_parametertypes.jl b/test/test_parametertypes.jl index c785c161e..cd5f04021 100644 --- a/test/test_parametertypes.jl +++ b/test/test_parametertypes.jl @@ -84,8 +84,8 @@ extpars = external_params(m.mi.md) @test isa(extpars[:e], ArrayModelParameter) @test isa(extpars[:f], ScalarModelParameter) # note that :f is stored as a scalar parameter even though its values are an array -@test typeof(extpars[:a].values) == TimestepMatrix{FixedTimestep{2000, 1, 2100}, arrtype, 1} -@test typeof(extpars[:b].values) == TimestepVector{FixedTimestep{2000, 1, 2100}, arrtype} +@test typeof(extpars[:a].values) == TimestepMatrix{FixedTimestep{2000, 1, 2100}, arrtype, 1, Array{arrtype, 2}} +@test typeof(extpars[:b].values) == TimestepVector{FixedTimestep{2000, 1, 2100}, arrtype, Array{arrtype, 1}} @test typeof(extpars[:c].values) == Array{arrtype, 1} @test typeof(extpars[:d].value) == numtype @@ -111,7 +111,7 @@ update_param!(m, :e, [4,5,6,7]) @test length(extpars) == 9 # The old dictionary has the default values that were added during build, so it has more entries @test length(new_extpars) == 6 -@test typeof(new_extpars[:a].values) == TimestepMatrix{FixedTimestep{2000, 1, 2100}, arrtype, 1} +@test typeof(new_extpars[:a].values) == TimestepMatrix{FixedTimestep{2000, 1, 2100}, arrtype, 1, Array{arrtype, 2}} @test typeof(new_extpars[:d].value) == numtype @test typeof(new_extpars[:e].values) == Array{arrtype, 1} From c7f1b8e8b7ea7ef788197c14324a64847e1fa2bc Mon Sep 17 00:00:00 2001 From: lrennels Date: Mon, 1 Mar 2021 23:28:15 -0800 Subject: [PATCH 30/32] sSimplify value replacement for TimestepArrays --- src/core/connections.jl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/core/connections.jl b/src/core/connections.jl index 028e24e5f..cd8899844 100644 --- a/src/core/connections.jl +++ b/src/core/connections.jl @@ -552,11 +552,7 @@ function _update_array_param!(obj::AbstractCompositeComponentDef, name, value) new_timestep_array = get_timestep_array(obj, T, N, ti, value) set_external_param!(obj, name, ArrayModelParameter(new_timestep_array, dim_names(param))) else - # DESIGN DISCUSSION - this became necessary when the Array type for - # TimestepArray widened to include SubArray, can check if it's necessary - # using test_parametertypes.jl around line 141 - T = eltype(param.values) - param.values.data = Array{T,N}(value) + param.values.data = value end else param.values = value From 01ea91cb546d428e579fa12000e5415d712e5067 Mon Sep 17 00:00:00 2001 From: lrennels Date: Sat, 6 Mar 2021 18:30:47 -0800 Subject: [PATCH 31/32] Add composites to datum_size and get_span methods --- src/core/defs.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/defs.jl b/src/core/defs.jl index c8db6e026..efafdd47f 100644 --- a/src/core/defs.jl +++ b/src/core/defs.jl @@ -252,7 +252,7 @@ function check_parameter_dimensions(md::ModelDef, value::AbstractArray, dims::Ve end # TBD: is this needed for composites? -function datum_size(obj::AbstractCompositeComponentDef, comp_def::ComponentDef, datum_name::Symbol) +function datum_size(obj::AbstractCompositeComponentDef, comp_def::AbstractComponentDef, datum_name::Symbol) dims = dim_names(comp_def, datum_name) if dims[1] == :time time_length = getspan(obj, comp_def)[1] @@ -673,7 +673,7 @@ function getspan(obj::AbstractComponentDef, comp_name::Symbol) return getspan(obj, comp_def) end -function getspan(obj::AbstractCompositeComponentDef, comp_def::ComponentDef) +function getspan(obj::AbstractCompositeComponentDef, comp_def::AbstractComponentDef) first = first_period(obj, comp_def) last = last_period(obj, comp_def) times = time_labels(obj) From cbe3f35abf97a02d5898a0af26f216acfb1beee3 Mon Sep 17 00:00:00 2001 From: lrennels Date: Sun, 7 Mar 2021 00:31:36 -0800 Subject: [PATCH 32/32] Put offset back into connector components --- src/core/connections.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/connections.jl b/src/core/connections.jl index cd8899844..52231b364 100644 --- a/src/core/connections.jl +++ b/src/core/connections.jl @@ -623,9 +623,10 @@ function add_connector_comps!(obj::AbstractCompositeComponentDef) # add a connection between ConnectorComp and the external backup data add_external_param_conn!(obj, ExternalParameterConnection(conn_path, :input2, conn.backup)) - # TBD: first/last stuff may be deprecated + # set the first and last parameters for WITHIN the component which + # decide when backup is used and when connectin is used src_comp_def = compdef(obj, conn.src_comp_path) - set_param!(obj, conn_comp_name, :first, first_period(obj, src_comp_def)) + set_param!(obj, conn_comp_name, :first, first_period(obj, src_comp_def) + conn.offset) set_param!(obj, conn_comp_name, :last, last_period(obj, src_comp_def)) end end