Improved logging with MTS

dispaset/__init__.py

@@ -3,6 +3,13 @@
 
 from .common import commons
 
+# Remove old log file:
+for filename in (f for f in os.listdir('.') if f.endswith('.dispa.log')):
+    try:
+        os.remove(filename)
+    except OSError:
+        print ('Could not erase previous log file ' + filename)
+
 # Logging: # TODO: Parametrize in dispacli or external config
 _LOGCONFIG = {
      "version": 1,
@@ -84,9 +91,4 @@
 
 from .cli import *
 
-# Remove old log file:
-for filename in (f for f in os.listdir('.') if f.endswith('.dispa.log')):
-    try:
-        os.remove(filename)
-    except OSError:
-        print ('Could not erase previous log file ' + filename)
+

dispaset/preprocessing/build.py

@@ -728,8 +728,5 @@ def build_single_run(config, profiles=None):
         with open(os.path.join(sim, 'Inputs.p'), 'wb') as pfile:
             pickle.dump(SimData, pfile, protocol=pickle.HIGHEST_PROTOCOL)
     logging.info('Build finished')
-
-    if os.path.isfile(commons['logfile']):
-        shutil.copy(commons['logfile'], os.path.join(sim, 'warn_preprocessing.log'))
 
     return SimData

dispaset/preprocessing/preprocessing.py

@@ -8,6 +8,7 @@
 import datetime as dt
 import logging
 import sys
+import os, shutil
 
 import pandas as pd
 import numpy as np
@@ -18,14 +19,7 @@
 from ..misc.gdx_handler import gdx_to_dataframe, gdx_to_list
 from ..postprocessing.data_handler import GAMSstatus
 from .utils import pd_timestep
-
-try:
-    from future.builtins import int
-except ImportError:
-    logging.warning(
-        "Couldn't import future package. Numeric operations may differ among different versions due to incompatible "
-        "variable types")
-    pass
+from ..common import commons
 
 
 def build_simulation(config, mts_plot=None, MTSTimeStep=None):
@@ -53,7 +47,14 @@ def build_simulation(config, mts_plot=None, MTSTimeStep=None):
     else:
         new_profiles = mid_term_scheduling(config, mts_plot=mts_plot, TimeStep=MTSTimeStep)
         # Build simulation data with new profiles
+        logging.info('\n\nBuilding final simulation\n')
         SimData = build_single_run(config, new_profiles)
+
+    # Copy the log to the simulation folder:
+    if os.path.isfile(commons['logfile']):
+        shutil.copy(commons['logfile'], os.path.join(config['SimulationDirectory'], commons['logfile']))
+    else:
+        logging.error('Could not find log file in current directory')
     return SimData
 
 def _check_results(results):
@@ -139,7 +140,7 @@ def mid_term_scheduling(config, TimeStep=None, mts_plot=None):
         temp_results = {}
         profiles = pd.DataFrame(index=idx)
         for i, c in enumerate(config['mts_zones']):
-            logging.info('(Currently simulating Zone): ' + str(i + 1) + ' out of ' + str(no_of_zones))
+            logging.info('\n\nLaunching Mid-Term Scheduling for zone '+ c + ' (Number ' + str(i + 1) + ' out of ' + str(no_of_zones) + ')\n')
             temp_config['zones'] = [c]  # Override zone that needs to be simulated
             SimData = build_single_run(temp_config)  # Create temporary SimData
             units = SimData['units']
@@ -172,6 +173,7 @@ def mid_term_scheduling(config, TimeStep=None, mts_plot=None):
 
     # Solving reservoir levels for all regions simultaneously
     elif config['HydroScheduling'] == 'Regional':
+        logging.info('\n\nLaunching regional Mid-Term Scheduling \n')
         SimData = build_single_run(temp_config)  # Create temporary SimData
         units = SimData['units']
         r = solve_GAMS(sim_folder=temp_config['SimulationDirectory'],