diff --git a/Project.toml b/Project.toml index 69b3baaf2..e55823bb4 100644 --- a/Project.toml +++ b/Project.toml @@ -40,6 +40,7 @@ julia = "1" IteratorInterfaceExtensions = "^0.1.1, ^1" TableTraits = "^0.4.1, ^1" Electron = "0.2, 0.3, 0.4, 1" +DataFrames = "0.19.1" [extras] DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" diff --git a/src/explorer/buildspecs.jl b/src/explorer/buildspecs.jl index fadf21797..9886a0d56 100644 --- a/src/explorer/buildspecs.jl +++ b/src/explorer/buildspecs.jl @@ -305,13 +305,13 @@ function getdatapart(df, dffields, plottype::Symbol) # loop over rows and create a dictionary for each row if plottype == :multiline - cols = (df[1], df[2], df[3]) + cols = (df[!, 1], df[!, 2], df[!, 3]) datastring = getmultiline(cols, dffields) elseif plottype == :line - cols = (df[1], df[2]) + cols = (df[!, 1], df[!, 2]) datastring = getline(cols, dffields) else - cols = (df[1], df[2]) + cols = (df[!, 1], df[!, 2]) datastring = getbar(cols, dffields) end diff --git a/src/mcs/montecarlo.jl b/src/mcs/montecarlo.jl index a90f50a2c..7bb38c98c 100644 --- a/src/mcs/montecarlo.jl +++ b/src/mcs/montecarlo.jl @@ -63,7 +63,7 @@ function _store_param_results(m::Model, datum_key::Tuple{Symbol, Symbol}, trialn else trial_df = getdataframe(m, comp_name, datum_name) - trial_df[:trialnum] = trialnum + trial_df[!, :trialnum] .= trialnum # println("size of trial_df: $(size(trial_df))") if haskey(results, datum_key) diff --git a/src/utils/getdataframe.jl b/src/utils/getdataframe.jl index a12f99d1c..00aaa8fea 100644 --- a/src/utils/getdataframe.jl +++ b/src/utils/getdataframe.jl @@ -15,7 +15,7 @@ function _load_dataframe(m::Model, comp_name::Symbol, item_name::Symbol, df::Uni # Create a new df if one was not passed in df = df === nothing ? DataFrame() : df - if haskey(df, item_name) + if hasproperty(df, item_name) error("An item named $item_name already exists in this DataFrame") end @@ -29,7 +29,7 @@ function _load_dataframe(m::Model, comp_name::Symbol, item_name::Symbol, df::Uni if num_dims == 1 dim1name = dims[1] dim1 = dimension(md, dim1name) - df[dim1name] = collect(keys(dim1)) + df[!, dim1name] = collect(keys(dim1)) # @info "dim: $dim1name size(df): $(size(df))" # df[item_name] = data @@ -44,9 +44,9 @@ function _load_dataframe(m::Model, comp_name::Symbol, item_name::Symbol, df::Uni data[1:(last-first+1)], # ignore padding after these values repeat([missing], inner=length(dim1) - last)) # @info "len shifted: $(length(shifted_data))" - df[item_name] = shifted_data + df[!, item_name] = shifted_data else - df[item_name] = data + df[!, item_name] = data end else df = _df_helper(m, comp_name, item_name, dims, data) @@ -72,8 +72,8 @@ function _df_helper(m::Model, comp_name::Symbol, item_name::Symbol, dims::Vector keys2 = collect(keys(dim2)) len_dim2 = length(dim2) - df[dim1name] = repeat(keys1, inner = [len_dim2]) - df[dim2name] = repeat(keys2, outer = [len_dim1]) + df[!, dim1name] = repeat(keys1, inner = [len_dim2]) + df[!, dim2name] = repeat(keys2, outer = [len_dim1]) if dim1name == :time && size(data)[1] != len_dim1 #length(time_labels(md)) ci = compinstance(m.mi, comp_name) @@ -86,7 +86,7 @@ function _df_helper(m::Model, comp_name::Symbol, item_name::Symbol, dims::Vector data = vcat(top, data, bottom) end - df[item_name] = cat([vec(data[i, :]) for i = 1:len_dim1]...; dims=1) + df[!, item_name] = cat([vec(data[i, :]) for i = 1:len_dim1]...; dims=1) else # shift the data to be padded with missings if this data is shorter than the model @@ -109,14 +109,14 @@ function _df_helper(m::Model, comp_name::Symbol, item_name::Symbol, dims::Vector for i in 1:size(data)[1] indexes[1] = i subdf = _df_helper(m, comp_name, item_name, dims[2:end], data[indexes...]) - subdf[dims[1]] = keys1[i] + subdf[!, dims[1]] .= keys1[i] if i == 1 # add required columns in the first iteration df_names = names(df) for name in names(subdf) if ! (name in df_names) - df[name] = [] + df[!, name] = [] end end end @@ -147,7 +147,7 @@ function getdataframe(m::Model, pairs::Pair{Symbol, Symbol}...) error("Can't create DataFrame from items with different dimensions ($comp_name1.$item_name1: $dims vs $comp_name.$item_name: $next_dims)") end result = getdataframe(m, comp_name, item_name) - df = hcat(df, result[[item_name]]) # [[xx]] retrieves a 1 column DataFrame + df = hcat(df, result[!, [item_name]]) # [[xx]] retrieves a 1 column DataFrame end return df diff --git a/test/dependencies/Manifest.toml b/test/dependencies/Manifest.toml index 09b99e44f..317008093 100644 --- a/test/dependencies/Manifest.toml +++ b/test/dependencies/Manifest.toml @@ -350,7 +350,6 @@ version = "0.6.1" deps = ["CSVFiles", "Compose", "DataFrames", "DataStructures", "Dates", "DelimitedFiles", "Distributions", "Electron", "ExcelFiles", "FileIO", "FilePaths", "GlobalSensitivityAnalysis", "GraphPlot", "IterTools", "IterableTables", "JSON", "LightGraphs", "LinearAlgebra", "Logging", "MacroTools", "MetaGraphs", "NamedArrays", "Pkg", "ProgressMeter", "Query", "Random", "Serialization", "Statistics", "StatsBase", "StringBuilders", "TableTraits", "VegaLite"] path = "../.." uuid = "e4e893b0-ee5e-52ea-8111-44b3bdec128c" -version = "0.8.7-DEV" [[MimiFUND]] deps = ["Dates", "DelimitedFiles", "Distributions", "Mimi", "StatsBase"] diff --git a/test/mcs/test_defmcs.jl b/test/mcs/test_defmcs.jl index dea1de722..da31ecaa4 100644 --- a/test/mcs/test_defmcs.jl +++ b/test/mcs/test_defmcs.jl @@ -82,7 +82,7 @@ function show_E_Global(year::Int; bins=40) @select i @collect DataFrame end - histogram(df[:E_Global], bins=bins, + histogram(df.E_Global, bins=bins, title="Distribution of global emissions in $year", xlabel="Emissions") end diff --git a/test/test_getdataframe.jl b/test/test_getdataframe.jl index 149be6cc3..4dd058b72 100644 --- a/test/test_getdataframe.jl +++ b/test/test_getdataframe.jl @@ -60,12 +60,12 @@ run(model1) # Regular getdataframe df = getdataframe(model1, :testcomp1=>:var1, :testcomp1=>:par1, :testcomp2=>:var2, :testcomp2=>:par2) dim = Mimi.dimension(model1, :time) -@test df[:var1] == df[:par1] == years -@test all(ismissing, df[:var2][1 : dim[late_first]-1]) -@test all(ismissing, df[:par2][1 : dim[late_first]-1]) -@test df[:var2][dim[late_first] : dim[early_last]] == df[:par2][dim[late_first] : dim[early_last]] == late_first:5:early_last -@test all(ismissing, df[:var2][dim[years[end]] : dim[early_last]]) -@test all(ismissing, df[:par2][dim[years[end]] : dim[early_last]]) +@test df.var1 == df.par1 == years +@test all(ismissing, df.var2[1 : dim[late_first]-1]) +@test all(ismissing, df.par2[1 : dim[late_first]-1]) +@test df.var2[dim[late_first] : dim[early_last]] == df.par2[dim[late_first] : dim[early_last]] == late_first:5:early_last +@test all(ismissing, df.var2[dim[years[end]] : dim[early_last]]) +@test all(ismissing, df.par2[dim[years[end]] : dim[early_last]]) # Test trying to load an item into an existing dataframe where that item key already exists @test_throws ErrorException _load_dataframe(model1, :testcomp1, :var1, df) @@ -123,7 +123,7 @@ df3 = getdataframe(model3, :testcomp3=>:par3) @test size(df3) == (length(rates)*length(regions)*length(years), 4) # Test that times outside the component's time span are padded with `missing` values -@test all(ismissing, df3[:par3][1 : (length(rates)*length(regions)*(dim[late_first]-1))]) -@test all(ismissing, df3[:par3][end - (length(rates)*length(regions)*(dim[end]-dim[early_last]))+1: end]) +@test all(ismissing, df3.par3[1 : (length(rates)*length(regions)*(dim[late_first]-1))]) +@test all(ismissing, df3.par3[end - (length(rates)*length(regions)*(dim[end]-dim[early_last]))+1: end]) end #module diff --git a/test/test_mult_getdataframe.jl b/test/test_mult_getdataframe.jl index 3f766eba6..71516b2bc 100644 --- a/test/test_mult_getdataframe.jl +++ b/test/test_mult_getdataframe.jl @@ -164,11 +164,11 @@ run1 = run_my_model() expanded_get_dataframe = getdataframe(run1, :grosseconomy =>(:YGROSS, :K)) curr_get_dataframe = getdataframe(run1, :grosseconomy, :YGROSS) -@test(expanded_get_dataframe[3] == curr_get_dataframe[3]) +@test(expanded_get_dataframe[!, 3] == curr_get_dataframe[!, 3]) expanded_get_dataframe = getdataframe(run1, :grosseconomy => :YGROSS, :emissions => :E) curr_get_dataframe = getdataframe(run1, :emissions, :E) -@test(expanded_get_dataframe[4] == curr_get_dataframe[3]) +@test(expanded_get_dataframe[!, 4] == curr_get_dataframe[!, 3]) ################# @@ -196,7 +196,7 @@ run(my_model) #Regular getdataframe dataframe = getdataframe(my_model, :testcomp1 => :var1) -@test(dataframe[2] == par) +@test(dataframe[!, 2] == par) #Test trying to getdataframe from component that does not exist @test_throws ErrorException getdataframe(my_model, :testcomp1 => :var2)