Remove duplicate code in solve.py

dispaset/__init__.py

@@ -52,7 +52,7 @@
 from .preprocessing.data_handler import load_config_excel, load_config_yaml, load_config, export_yaml_config
 from .preprocessing.preprocessing import build_simulation, adjust_capacity, adjust_storage, get_temp_sim_results, mid_term_scheduling, build_full_simulation
 
-from .solve import solve_GAMS, solve_GAMS_simple, solve_pyomo
+from .solve import solve_GAMS, solve_pyomo
 
 from .postprocessing.data_handler import get_sim_results, ds_to_df
 from .postprocessing.postprocessing import get_result_analysis, get_indicators_powerplant, aggregate_by_fuel, CostExPost

dispaset/misc/gms_handler.py

@@ -6,16 +6,16 @@
 
 from .str_handler import force_str
 
-def solve_high_level(gams_folder,sim_folder,output_lst=False):
+def solve_high_level(gams_folder, sim_folder, gams_file='UCM_h.gms', result_file='Results.gdx', output_lst=False):
     """Use higher level apis to run GAMSy"""
     # create GAMS workspace:
     try:
         from gams import GamsWorkspace
         ws = GamsWorkspace(system_directory=str(gams_folder), debug=3)
-        shutil.copy(os.path.join(sim_folder, 'UCM_h.gms'), ws.working_directory)
+        shutil.copy(os.path.join(sim_folder, gams_file), ws.working_directory)
         shutil.copy(os.path.join(sim_folder, 'Inputs.gdx'), ws.working_directory)
         shutil.copy(os.path.join(sim_folder, 'cplex.opt'), ws.working_directory)
-        t1 = ws.add_job_from_file('UCM_h.gms')
+        t1 = ws.add_job_from_file(gams_file)
         opt = ws.add_options()
         # Do not create .lst file
         if not output_lst:
@@ -33,23 +33,27 @@ def solve_high_level(gams_folder,sim_folder,output_lst=False):
             logging.error('The following error occured when trying to solve the model in gams: ' + str(e))
             sys.exit(1)
     # copy the result file to the simulation environment folder:
-    shutil.copy(os.path.join(ws.working_directory, 'Results.gdx'), sim_folder)
-    for filename in ['UCM_h.lst','UCM_h.log','debug.gdx']:
+    shutil.copy(os.path.join(ws.working_directory, result_file), sim_folder)
+    gams_file_base = os.path.splitext(gams_file)[0]
+    for filename in [gams_file_base + '.lst', gams_file_base + '.log', 'debug.gdx']:
         if os.path.isfile(os.path.join(ws.working_directory, filename)):
             shutil.copy(os.path.join(ws.working_directory, filename), sim_folder)
     logging.info('Completed simulation in {0:.2f} seconds'.format(time.time() - time0))
     return status
 
 
-def solve_low_level(gams_folder,sim_folder,output_lst=False,logoption=3):
-    """Use lower level apis to run GAMS. BAsed on GAMS xpexample2.py"""
+def solve_low_level(gams_folder, sim_folder, gams_file='UCM_h.gms', result_file='Results.gdx', output_lst=False, logoption=3):
+    """Use lower level apis to run GAMS. Based on GAMS xpexample2.py.
+     We use the same signature as in solve_high_level for consistency when we call it.
+     As we define the working directory to be the simulation directory we do not need to define the output names
+     and move them around the filesystem"""
     from gamsxcc import new_gamsxHandle_tp, gamsxRunExecDLL, gamsxFree, gamsxCreateD, GMS_SSSIZE
     from optcc import new_optHandle_tp, optReadDefinition, optSetStrStr, optSetIntStr, optHandleToPtr, optFree, \
         optCreateD
 
     sim_folder = force_str(sim_folder)
     gams_folder = force_str(gams_folder)
-    model = os.path.abspath(os.path.join(sim_folder, 'UCM_h.gms'))
+    model = os.path.abspath(os.path.join(sim_folder, gams_file))
 
     def callGams(gamsxHandle, optHandle, sysDir, model):
         deffile = force_str(os.path.join(sysDir, 'optgams.def'))
@@ -74,86 +78,6 @@ def callGams(gamsxHandle, optHandle, sysDir, model):
     optHandle = new_optHandle_tp()
     gamsxHandle = new_gamsxHandle_tp()
 
-    try:
-        __ = gamsxCreateD(gamsxHandle, gams_folder, GMS_SSSIZE)
-        __ = optCreateD(optHandle, gams_folder, GMS_SSSIZE)
-        time0 = time.time()
-        status = callGams(gamsxHandle, optHandle, gams_folder, model)
-    except Exception as e:
-        logging.error('The following error occured when trying to solve the model in gams: ' + str(e))
-        return False
-    finally:
-        optFree(optHandle)
-        gamsxFree(gamsxHandle)
-    logging.info('Completed simulation in {0:.2f} seconds'.format(time.time() - time0))
-    return status
-
-# Simple GAMMS model
-def solve_high_level_simple(gams_folder,sim_folder,output_lst=False):
-    """Use higher level apis to run GAMSy"""
-    # create GAMS workspace:
-    try:
-        from gams import GamsWorkspace
-        ws = GamsWorkspace(system_directory=str(gams_folder), debug=3)
-        shutil.copy(os.path.join(sim_folder, 'UCM_h_simple.gms'), ws.working_directory)
-        shutil.copy(os.path.join(sim_folder, 'Inputs.gdx'), ws.working_directory)
-        shutil.copy(os.path.join(sim_folder, 'cplex.opt'), ws.working_directory)
-        t1 = ws.add_job_from_file('UCM_h_simple.gms')
-        opt = ws.add_options()
-        # Do not create .lst file
-        if not output_lst:
-            if sys.platform == 'win32':
-                opt.output = 'nul'
-            else:
-                opt.output = '/dev/null'
-        time0 = time.time()
-        status = t1.run(opt)
-    except Exception as e:
-        if 'optCreateD' in str(e):
-            logging.error('The GAMS solver can only be run once in the same console. Please open another console')
-            sys.exit(1)
-        else:
-            logging.error('The following error occured when trying to solve the model in gams: ' + str(e))
-            sys.exit(1)
-    # copy the result file to the simulation environment folder:
-    shutil.copy(os.path.join(ws.working_directory, 'Results_simple.gdx'), sim_folder)
-    for filename in ['UCM_h_simple.lst','UCM_h_simple.log','debug.gdx']:
-        if os.path.isfile(os.path.join(ws.working_directory, filename)):
-            shutil.copy(os.path.join(ws.working_directory, filename), sim_folder)
-    logging.info('Completed simulation in {0:.2f} seconds'.format(time.time() - time0))
-    return status    
-
-def solve_low_level_simple(gams_folder,sim_folder,output_lst=False,logoption=3):
-    """Use lower level apis to run GAMS. BAsed on GAMS xpexample2.py"""
-    from gamsxcc import new_gamsxHandle_tp, gamsxRunExecDLL, gamsxFree, gamsxCreateD, GMS_SSSIZE
-    from optcc import new_optHandle_tp, optReadDefinition, optSetStrStr, optSetIntStr, optHandleToPtr, optFree, \
-        optCreateD
-
-    sim_folder = force_str(sim_folder)
-    gams_folder = force_str(gams_folder)
-    model = os.path.join(sim_folder, 'UCM_h_simple.gms')
-
-    def callGams(gamsxHandle, optHandle, sysDir, model):
-        deffile = force_str(sysDir + u'/optgams.def')
-
-        if optReadDefinition(optHandle, deffile):
-            logging.error("*** Error ReadDefinition, cannot read def file:" + deffile)
-            return False
-
-        optSetStrStr(optHandle, "SysDir", sysDir)
-        optSetStrStr(optHandle, "WorkDir", sim_folder)
-        optSetStrStr(optHandle, "Input", model)
-        optSetIntStr(optHandle, "LogOption", logoption)
-        ret = gamsxRunExecDLL(gamsxHandle, optHandleToPtr(optHandle), sysDir, 1)
-        if ret[0] != 0:
-            logging.error("*** Error RunExecDLL: Error in GAMS call = " + str(ret[1]))
-            return False
-
-        return True
-
-    optHandle = new_optHandle_tp()
-    gamsxHandle = new_gamsxHandle_tp()
-
     try:
         __ = gamsxCreateD(gamsxHandle, gams_folder, GMS_SSSIZE)
         __ = optCreateD(optHandle, gams_folder, GMS_SSSIZE)

dispaset/preprocessing/preprocessing.py

@@ -23,7 +23,7 @@
 from .utils import clustering, interconnections, incidence_matrix, select_units, EfficiencyTimeSeries
 from .data_handler import UnitBasedTable,NodeBasedTable,merge_series, define_parameter, load_time_series
 
-from ..solve import solve_GAMS_simple
+from ..solve import solve_GAMS
 
 from ..misc.gdx_handler import write_variables, gdx_to_list, gdx_to_dataframe
 from ..common import commons  # Load fuel types, technologies, timestep, etc:
@@ -909,7 +909,7 @@ def adjust_storage(inputs,tech_fuel,scaling=1,value=None,write_gdx=False,dest_pa
             os.remove('Inputs.gdx')
     return SimData
 
-def get_temp_sim_results(config, gams_dir=None, cache=False, temp_path='.pickle'):
+def get_temp_sim_results(config, gams_dir=None):
     """
     This function reads the simulation environment folder once it has been solved and loads
     the input variables together with the results.
@@ -962,8 +962,10 @@ def mid_term_scheduling(config, zones, profiles=None):
             temp_config = dict(config)
             temp_config['zones'] = [c]                    # Override zone that needs to be simmulated
             _ = build_simulation(temp_config)       # Create temporary SimData   
-            r = solve_GAMS_simple(temp_config['SimulationDirectory'], 
-                                  temp_config['GAMS_folder'])
+            r = solve_GAMS(sim_folder=temp_config['SimulationDirectory'],
+                           gams_folder=temp_config['GAMS_folder'],
+                           gams_file='UCM_h_simple.gms',
+                           result_file='Results_simple.gdx')
             temp_results[c] = get_temp_sim_results(config)
     #        print('Zones simulated: ' + str(i) + '/' + str(no_of_zones))
         temp = pd.DataFrame()
@@ -988,7 +990,10 @@ def mid_term_scheduling(config, zones, profiles=None):
             temp_config = dict(config)
             temp_config['zones'] = zones               # Override zones that need to be simmulated
         _ = build_simulation(temp_config)        # Create temporary SimData   
-        r = solve_GAMS_simple(temp_config['SimulationDirectory'], temp_config['GAMS_folder'])
+        r = solve_GAMS(sim_folder=temp_config['SimulationDirectory'],
+                       gams_folder=temp_config['GAMS_folder'],
+                       gams_file='UCM_h_simple.gms',
+                       result_file='Results_simple.gdx')
         temp_results = get_temp_sim_results(config)
         if 'OutputStorageLevel' not in temp_results:
             logging.critical('Storage levels in the selected region were not computed, please check that storage units '

dispaset/solve.py

@@ -45,7 +45,7 @@
 import time
 
 from .misc.gdx_handler import get_gams_path, import_local_lib, package_exists
-from .misc.gms_handler import solve_high_level, solve_low_level, solve_low_level_simple, solve_high_level_simple
+from .misc.gms_handler import solve_high_level, solve_low_level
 from .common import commons
 
 
@@ -75,7 +75,7 @@ def is_sim_folder_ok(sim_folder):
     return True
 
 
-def solve_GAMS(sim_folder, gams_folder=None, output_lst=False):
+def solve_GAMS(sim_folder, gams_folder=None, gams_file='UCM_h.gms', result_file='Results.gdx', output_lst=False):
     '''
     Function used to run the optimization using the GAMS engine.
 
@@ -92,7 +92,6 @@ def solve_GAMS(sim_folder, gams_folder=None, output_lst=False):
         solv_func = solve_high_level
         logging.info('Using the high-level gams api')
     else:
-        solv_func = solve_low_level
         logging.warning('Could not find the GAMS APIs. Trying to locate local version')
         if not import_local_lib('lowlevel'):
             return False
@@ -107,7 +106,7 @@ def solve_GAMS(sim_folder, gams_folder=None, output_lst=False):
         #Temporary warning for Spyder users:
         if any(['SPY_' in name for name in os.environ]): # check if spyder
             logging.info("\nIf the script seems stuck at this place \n(gams is optimizing but not output is displayed), \nit is preferable to run Dispa-SET in a \nseparate terminal (in Spyder: Preferences - Run - \nExecute in an external system terminal)")
-        ret = solv_func(gams_folder, sim_folder, output_lst=output_lst)
+        ret = solv_func(gams_folder, sim_folder, gams_file, result_file, output_lst=output_lst)
         if os.path.isfile(os.path.join(sim_folder, 'debug.gdx')):
             logging.warning('A debug file was created. There has probably been an optimization error')
         if os.path.isfile(commons['logfile']):
@@ -148,46 +147,3 @@ def solve_pyomo(sim_folder):
     if os.path.isfile(commons['logfile']):
         shutil.copy(commons['logfile'], os.path.join(sim_folder, 'warn_solve.log'))
     return results
-
-# Mid-term-scheduling GAMS model
-def solve_GAMS_simple(sim_folder, gams_folder=None, output_lst=False):
-    '''
-    Function used to run the optimization using the GAMS engine.
-
-    :param sim_folder: path to a valid Dispa-SET simulation folder
-    :param gams_folder: path to the gams folder. If not provided, the script will try to find it automatically
-    :param work_dir: path to the working directory (does not need to be provided)
-    :param output_lst: Set to True to conserve a copy of the GAMS lst file in the simulation folder
-    '''
-
-    if package_exists('gamsxcc') and package_exists('optcc'):
-        solv_func = solve_low_level_simple
-        logging.info('Using the low-level gams api')
-    elif package_exists('gams'):
-        solv_func = solve_high_level_simple
-        logging.info('Using the high-level gams api')
-    else:
-        solv_func = solve_high_level_simple
-        logging.warning('Could not import lower level APIs. Trying to locate local version')
-        if not import_local_lib('gams'):
-            return False
-    gams_folder = get_gams_path(gams_folder)
-    if not gams_folder:  # couldn't locate
-        logging.error('GAMS path cannot be located. Simulation is stopped')
-        return False
-    sim_folder = os.path.abspath(sim_folder)
-    gams_folder = os.path.abspath(gams_folder)
-
-    if is_sim_folder_ok(sim_folder):
-        #Temporary warning for Spyder users:
-        if any(['SPY_' in name for name in os.environ]): # check if spyder
-            logging.info("\nIf the script seems stuck at this place \n(gams is optimizing but not output is displayed), \nit is preferable to run Dispa-SET in a \nseparate terminal (in Spyder: Preferences - Run - \nExecute in an external system terminal)")
-        ret = solv_func(gams_folder, sim_folder, output_lst=output_lst)
-        if os.path.isfile(os.path.join(sim_folder, 'debug.gdx')):
-            logging.warning('A debug file was created. There has probably been an optimization error')
-        if os.path.isfile(commons['logfile']):
-            shutil.copy(commons['logfile'], os.path.join(sim_folder, 'warn_solve.log'))
-        return ret
-    else:
-        return False
-