Added strict flag to check all assets have partition data (#399)

src/io.jl

@@ -5,23 +5,26 @@ export create_energy_problem_from_csv_folder,
     compute_flows_partitions!
 
 """
-    energy_problem = create_energy_problem_from_csv_folder(input_folder)
+    energy_problem = create_energy_problem_from_csv_folder(input_folder; strict = false)
 
 Returns the [`TulipaEnergyModel.EnergyProblem`](@ref) reading all data from CSV files
 in the `input_folder`.
 This is a wrapper around `create_graph_and_representative_periods_from_csv_folder` that creates
 the `EnergyProblem` structure.
+Set strict = true to error if assets are missing from partition data.
 """
-function create_energy_problem_from_csv_folder(input_folder::AbstractString)
+function create_energy_problem_from_csv_folder(input_folder::AbstractString; strict = false)
     graph, representative_periods =
-        create_graph_and_representative_periods_from_csv_folder(input_folder)
+        create_graph_and_representative_periods_from_csv_folder(input_folder; strict = strict)
     return EnergyProblem(graph, representative_periods)
 end
 
 """
-    graph, representative_periods = create_graph_and_representative_periods_from_csv_folder(input_folder)
+    graph, representative_periods = create_graph_and_representative_periods_from_csv_folder(input_folder; strict = false)
 
 Returns the `graph` structure that holds all data, and the `representative_periods` array.
+Set strict = true to error if assets are missing from partition data.
+
 The following files are expected to exist in the input folder:
 
   - `assets-data.csv`: Following the [`TulipaEnergyModel.AssetData`](@ref) specification.
@@ -44,7 +47,10 @@ The returned structures are:
   - `representative_periods`: An array of
     [`TulipaEnergyModel.RepresentativePeriod`](@ref) ordered by their IDs.
 """
-function create_graph_and_representative_periods_from_csv_folder(input_folder::AbstractString)
+function create_graph_and_representative_periods_from_csv_folder(
+    input_folder::AbstractString;
+    strict = false,
+)
     # Read data
     fillpath(filename) = joinpath(input_folder, filename)
 
@@ -57,6 +63,20 @@ function create_graph_and_representative_periods_from_csv_folder(input_folder::A
     assets_partitions_df = read_csv_with_schema(fillpath("assets-partitions.csv"), AssetPartitionData)
     flows_partitions_df  = read_csv_with_schema(fillpath("flows-partitions.csv"), FlowPartitionData)
 
+    # Error if partition data is missing assets (if strict)
+    if strict
+        missing_assets = setdiff(assets_data_df[!, "name"], assets_partitions_df[!, "asset"])
+        if length(missing_assets) > 0
+            msg = "Error: Partition data missing for these assets: \n"
+            for a in missing_assets
+                msg *= "- $a\n"
+            end
+            msg *= "To assume missing asset resolutions follow the representative period's time resolution, set strict = false.\n"
+
+            error(msg)
+        end
+    end
+
     # Sets and subsets that depend on input data
 
     # TODO: Depending on the outcome of issue #294, this can be done more efficiently with DataFrames, e.g.,
@@ -125,7 +145,7 @@ function create_graph_and_representative_periods_from_csv_folder(input_folder::A
         )
     end
 
-    for rp_id = 1:length(representative_periods), a in labels(graph)
+    for rp_id ∈ 1:length(representative_periods), a in labels(graph)
         # Get all profile data for asset=a and rp=rp_id
         matching = (assets_profiles_df.asset .== a) .& (assets_profiles_df.rep_period_id .== rp_id)
         if sum(matching) == 0
@@ -135,7 +155,7 @@ function create_graph_and_representative_periods_from_csv_folder(input_folder::A
         graph[a].profiles[rp_id] = profile_data
     end
 
-    for rp_id = 1:length(representative_periods), (u, v) in edge_labels(graph)
+    for rp_id ∈ 1:length(representative_periods), (u, v) in edge_labels(graph)
         # Get all profile data for flow=(u,v) and rp=rp_id
         matching =
             (flows_profiles_df.from_asset .== u) .&
@@ -287,7 +307,7 @@ function _parse_rp_partition end
 
 function _parse_rp_partition(::Val{:uniform}, time_step_string, rp_time_steps)
     duration = parse(Int, time_step_string)
-    partition = [i:i+duration-1 for i = 1:duration:length(rp_time_steps)]
+    partition = [i:i+duration-1 for i ∈ 1:duration:length(rp_time_steps)]
     @assert partition[end][end] == length(rp_time_steps)
     return partition
 end
@@ -311,7 +331,7 @@ function _parse_rp_partition(::Val{:math}, time_step_string, rp_time_steps)
     block_instruction = split(time_step_string, "+")
     for R in block_instruction
         num, len = parse.(Int, split(R, "x"))
-        for _ = 1:num
+        for _ ∈ 1:num
             block = (1:len) .+ (block_begin - 1)
             block_begin += len
             push!(partition, block)
@@ -344,7 +364,7 @@ function compute_assets_partitions!(partitions, df, a, representative_periods)
         partitions[rp_id] = if j === nothing
             N = length(rp.time_steps)
             # If there is no time block specification, use default of 1
-            [k:k for k = 1:N]
+            [k:k for k ∈ 1:N]
         else
             _parse_rp_partition(Val(df[j, :specification]), df[j, :partition], rp.time_steps)
         end
@@ -374,7 +394,7 @@ function compute_flows_partitions!(partitions, df, u, v, representative_periods)
         partitions[rp_id] = if j === nothing
             N = length(rp.time_steps)
             # If there is no time block specification, use default of 1
-            [k:k for k = 1:N]
+            [k:k for k ∈ 1:N]
         else
             _parse_rp_partition(Val(df[j, :specification]), df[j, :partition], rp.time_steps)
         end

test/test-io.jl

@@ -6,6 +6,14 @@
             TulipaEnergyModel.AssetData,
         )
     end
+
+    @testset "Check missing asset partition if strict" begin
+        dir = joinpath(INPUT_FOLDER, "Norse")
+        @test_throws Exception TulipaEnergyModel.create_energy_problem_from_csv_folder(
+            dir,
+            strict = true,
+        )
+    end
 end
 
 @testset "Output validation" begin
@@ -52,12 +60,12 @@ end
         expected = Dict(
             (1, 1) => [1:3, 4:6, 7:9, 10:12],
             (2, 1) => [1:4, 5:8, 9:12],
-            (3, 1) => [i:i for i = 1:12],
-            (1, 2) => [i:i for i = 1:24],
+            (3, 1) => [i:i for i ∈ 1:12],
+            (1, 2) => [i:i for i ∈ 1:24],
             (2, 2) => [1:4, 5:8, 9:12, 13:15, 16:18, 19:21, 22:24],
             (3, 2) => [1:2, 3:4, 5:7, 8:10, 11:14, 15:18, 19:24],
         )
-        for a = 1:3, rp = 1:2
+        for a ∈ 1:3, rp ∈ 1:2
             @test dummy[a][rp] == expected[(a, rp)]
         end
     end
@@ -82,12 +90,12 @@ end
         expected = Dict(
             ((1, 2), 1) => [1:3, 4:6, 7:9, 10:12],
             ((2, 3), 1) => [1:4, 5:8, 9:12],
-            ((3, 4), 1) => [i:i for i = 1:12],
-            ((1, 2), 2) => [i:i for i = 1:24],
+            ((3, 4), 1) => [i:i for i ∈ 1:12],
+            ((1, 2), 2) => [i:i for i ∈ 1:24],
             ((2, 3), 2) => [1:4, 5:8, 9:12, 13:15, 16:18, 19:21, 22:24],
             ((3, 4), 2) => [1:2, 3:4, 5:7, 8:10, 11:14, 15:18, 19:24],
         )
-        for f in flows, rp = 1:2
+        for f in flows, rp ∈ 1:2
             @test dummy[f][rp] == expected[(f, rp)]
         end
     end