Merge pull request #934 from blue-marble/develop

GridPath v0.14.1

gridpath/auxiliary/import_export_rules.py

@@ -0,0 +1,74 @@
+# Copyright 2016-2022 Blue Marble Analytics LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+from pyomo.environ import value
+
+
+# Import-export rules
+
+# Export & import if USE is found only
+def export_rule_use(instance, quiet):
+    unserved_energy_found = any(
+        [
+            value(instance.Unserved_Energy_MW_Expression[z, tmp])
+            for z in getattr(instance, "LOAD_ZONES")
+            for tmp in getattr(instance, "TMPS")
+        ]
+    )
+
+    if unserved_energy_found:
+        if not quiet:
+            print("unserved energy found; exporting results")
+
+    return unserved_energy_found
+
+
+def summarize_results_use(scenario_directory, subproblem, stage, quiet):
+    if os.path.exists(
+        os.path.join(
+            scenario_directory,
+            subproblem,
+            stage,
+            "results",
+            "load_balance.csv",
+        )
+    ):
+        return True
+    else:
+        if not quiet:
+            print("skipping results summary")
+        return False
+
+
+def import_rule_use(results_directory, quiet):
+    if os.path.exists(os.path.join(results_directory, "load_balance.csv")):
+        import_results = True
+        if not quiet:
+            print("unserved energy found -- importing")
+    else:
+        import_results = False
+        if not quiet:
+            print("no unserved energy -- skipping")
+
+    return import_results
+
+
+import_export_rules = {
+    "USE": {
+        "export": export_rule_use,
+        "summarize": summarize_results_use,
+        "import": import_rule_use,
+    }
+}

gridpath/common_functions.py

@@ -140,7 +140,7 @@ def get_db_parser():
     return parser
 
 
-def get_parallel_get_inputs_parser():
+def get_get_inputs_parser():
     """ """
 
     parser = ArgumentParser(add_help=False)
@@ -153,18 +153,7 @@ def get_parallel_get_inputs_parser():
     return parser
 
 
-def get_parallel_solve_parser():
-    """ """
-
-    parser = ArgumentParser(add_help=False)
-    parser.add_argument(
-        "--n_parallel_solve", default=1, help="Solve n subproblems in parallel."
-    )
-
-    return parser
-
-
-def get_solve_parser():
+def get_run_scenario_parser():
     """
     Create ArgumentParser object which has the common set of arguments for
     solving a scenario (see run_scenario.py and run_end_to_end.py).
@@ -237,6 +226,29 @@ def get_solve_parser():
         help="Flag for test suite runs. Results not saved.",
     )
 
+    # Parallel solve
+    parser.add_argument(
+        "--n_parallel_solve",
+        default=1,
+        help="Solve n subproblems in parallel.",
+    )
+
+    # Results export rule name
+    parser.add_argument(
+        "--results_export_rule",
+        help="The name of the rule to use to decide whether to export results.",
+    )
+
+    return parser
+
+
+def get_import_results_parser():
+    parser = ArgumentParser(add_help=False)
+    parser.add_argument(
+        "--results_import_rule",
+        help="The name of the rule to use to decide whether to import results.",
+    )
+
     return parser
 
 

gridpath/get_scenario_inputs.py

@@ -36,7 +36,7 @@
     create_directory_if_not_exists,
     get_db_parser,
     get_required_e2e_arguments_parser,
-    get_parallel_get_inputs_parser,
+    get_get_inputs_parser,
 )
 from gridpath.auxiliary.module_list import determine_modules, load_modules
 from gridpath.auxiliary.scenario_chars import (
@@ -303,7 +303,7 @@ def parse_arguments(args):
         parents=[
             get_db_parser(),
             get_required_e2e_arguments_parser(),
-            get_parallel_get_inputs_parser(),
+            get_get_inputs_parser(),
         ],
     )
 

gridpath/import_scenario_results.py

@@ -22,26 +22,25 @@
 """
 
 from argparse import ArgumentParser
-import csv
 import os.path
 import pandas as pd
 import sys
 
 from gridpath.auxiliary.db_interface import get_scenario_id_and_name
+from gridpath.auxiliary.import_export_rules import import_export_rules
 from gridpath.common_functions import (
     determine_scenario_directory,
     get_db_parser,
     get_required_e2e_arguments_parser,
+    get_import_results_parser,
 )
 from db.common_functions import connect_to_database, spin_on_database_lock
 from db.utilities.scenario import delete_scenario_results
 from gridpath.auxiliary.module_list import determine_modules, load_modules
 from gridpath.auxiliary.scenario_chars import get_subproblem_structure_from_db
 
 
-def _import_rule(
-    db, scenario_id, subproblem, stage, results_directory, loaded_modules, quiet
-):
+def _import_rule(results_directory, quiet):
     """
     :return: boolean
 
@@ -216,15 +215,12 @@ def import_subproblem_stage_results_into_database(
     Import results for a subproblem/stage. We first check the import rule to
     determine whether to import.
     """
-    import_results = import_rule(
-        db=db,
-        scenario_id=scenario_id,
-        subproblem=subproblem,
-        stage=stage,
-        results_directory=results_directory,
-        loaded_modules=loaded_modules,
-        quiet=quiet,
-    )
+    if import_rule is None:
+        import_results = _import_rule(results_directory=results_directory, quiet=quiet)
+    else:
+        import_results = import_export_rules[import_rule]["import"](
+            results_directory=results_directory, quiet=quiet
+        )
 
     if import_results:
         c = db.cursor()
@@ -257,14 +253,19 @@ def parse_arguments(args):
     :return:
     """
     parser = ArgumentParser(
-        add_help=True, parents=[get_db_parser(), get_required_e2e_arguments_parser()]
+        add_help=True,
+        parents=[
+            get_db_parser(),
+            get_required_e2e_arguments_parser(),
+            get_import_results_parser(),
+        ],
     )
     parsed_arguments = parser.parse_known_args(args=args)[0]
 
     return parsed_arguments
 
 
-def main(import_rule, args=None):
+def main(args=None):
     """
 
     :return:
@@ -279,6 +280,7 @@ def main(import_rule, args=None):
     scenario_name_arg = parsed_arguments.scenario
     scenario_location = parsed_arguments.scenario_location
     quiet = parsed_arguments.quiet
+    import_rule = parsed_arguments.results_import_rule
 
     conn = connect_to_database(db_path=db_path)
     c = conn.cursor()
@@ -324,7 +326,7 @@ def main(import_rule, args=None):
 
     # Import appropriate results into database
     import_scenario_results_into_database(
-        import_rule=import_rule,
+        import_rule=parsed_arguments.results_import_rule,
         loaded_modules=loaded_modules,
         scenario_id=scenario_id,
         subproblems=subproblem_structure,
@@ -339,4 +341,4 @@ def main(import_rule, args=None):
 
 
 if __name__ == "__main__":
-    main(import_rule=_import_rule)
+    main()

gridpath/run_end_to_end.py

@@ -34,10 +34,9 @@
 from db.common_functions import connect_to_database, spin_on_database_lock
 from gridpath.common_functions import (
     get_db_parser,
-    get_solve_parser,
+    get_run_scenario_parser,
     get_required_e2e_arguments_parser,
-    get_parallel_get_inputs_parser,
-    get_parallel_solve_parser,
+    get_get_inputs_parser,
     create_logs_directory_if_not_exists,
     Logging,
     determine_scenario_directory,
@@ -48,6 +47,7 @@
     import_scenario_results,
     process_results,
 )
+from gridpath.run_scenario import _export_rule, _summarize_rule
 from gridpath.import_scenario_results import _import_rule
 from gridpath.auxiliary.db_interface import get_scenario_id_and_name
 
@@ -66,9 +66,8 @@ def parse_arguments(args):
         parents=[
             get_db_parser(),
             get_required_e2e_arguments_parser(),
-            get_solve_parser(),
-            get_parallel_get_inputs_parser(),
-            get_parallel_solve_parser(),
+            get_run_scenario_parser(),
+            get_get_inputs_parser(),
         ],
     )
 
@@ -374,7 +373,9 @@ def main(args=None):
         try:
             # make sure run_scenario.py gets the required --scenario argument
             run_scenario_args = args + ["--scenario", scenario]
-            expected_objective_values = run_scenario.main(args=run_scenario_args)
+            expected_objective_values = run_scenario.main(
+                args=run_scenario_args,
+            )
         except Exception as e:
             logging.exception(e)
             end_time = update_db_for_run_end(
@@ -395,7 +396,7 @@ def main(args=None):
 
     if not skip_import_results and not parsed_args.skip_import_results:
         try:
-            import_scenario_results.main(import_rule=_import_rule, args=args)
+            import_scenario_results.main(args=args)
         except Exception as e:
             logging.exception(e)
             end_time = update_db_for_run_end(