main_RL_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import envs\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from envs.custom_env_dir.dqn_agent import DQNAgent\n",
    "from envs.custom_env_dir.data_handler import DataHandler\n",
    "from envs.custom_env_dir.sup_model import SupModel\n",
    "from envs.custom_env_dir.utils import make_env\n",
    "from datetime import datetime\n",
    "import os \n",
    "import time\n",
    "\n",
    "\n",
    "''' THIS FUNCTION WILL BE CALLED FROM THE MAIN-METHOD BELOW'''\n",
    "def run_model(optimizer, gamma, lr, replace, hl, k, store_dir, mlp_hl, mlp_af, mlp_sl, input_dims, mlp, knn, n_BM, uncontrolled, obs, store_results, development, test, test_final):\n",
    "    \n",
    "    # Get collection of train, test, dev sets\n",
    "    train_collection, dev_collection, test_collection, train_count, dev_count, test_count, full_collection = DataHandler().get_data_7d_3split(include_weekends=True, \\\n",
    "                                                                                   n_episodes = 450, start_year=2018, start_month=10, start_day=1)\n",
    "\n",
    "    \n",
    "    # Define EV battery capacity in kWh\n",
    "    battery_capacity = 24\n",
    "    # Define residential EV charging rate in kW\n",
    "    charging_rate = 6\n",
    "    # Set penalty coefficient for incomplete charging\n",
    "    penalty_coefficient = 12\n",
    "    \n",
    "    # Get current directory to store model\n",
    "    cwd = os.getcwd()\n",
    "    \n",
    "    # Initialize best_score for tracking best model\n",
    "    best_score = -np.inf\n",
    "    \n",
    "    # Set parameters for testing\n",
    "    if test:\n",
    "        # Use development set for parameter tuning\n",
    "        if development:\n",
    "            test_collection = dev_collection\n",
    "            dataset = 'DEV'\n",
    "            \n",
    "        else:\n",
    "            dataset = 'TEST'\n",
    "            \n",
    "        # Create test environment and pass collection of days\n",
    "        env = gym.make('ChargingEnv-v0', game_collection=test_collection,\n",
    "                battery_capacity=battery_capacity, charging_rate=charging_rate,\n",
    "                penalty_coefficient=penalty_coefficient,obs=obs)\n",
    "        # Simulate each day in the set 10 times with different driving profiles\n",
    "        n_episodes = len(test_collection)*10\n",
    "        # Makes sure the agent does not learn during training\n",
    "        pre_train_steps = np.inf\n",
    "        # Load previously trained model\n",
    "        load_checkpoint = True\n",
    "        filename = dataset+'_'+optimizer+'_gamma'+str(gamma)+'_lr'+(('%.15f' % lr).rstrip('0').rstrip('.'))+'_replace'+str(replace)+'_HL'+str(hl) \n",
    "    \n",
    "    # Set parameters for training\n",
    "    else: \n",
    "        # Create training environment and pass collection of days\n",
    "        env = gym.make('ChargingEnv-v0', game_collection=train_collection,\n",
    "                battery_capacity=battery_capacity, charging_rate=charging_rate,\n",
    "                penalty_coefficient=penalty_coefficient,obs=obs)\n",
    "        # Do not load a checkpoint - train new model\n",
    "        load_checkpoint = False\n",
    "        # Train model for n_episodes episodes\n",
    "        n_episodes = 50000\n",
    "        # Specify number of random episodes before epsilon starts to decrease\n",
    "        pre_training_steps = 5000\n",
    "        \n",
    "        print('Train model for ' + str(n_episodes) + ' episodes with ' + str(pre_training_steps) + ' pre-train steps ...')\n",
    "        filename = 'TRAIN'+'_'+optimizer+'_gamma'+str(gamma)+'_lr'+(('%.15f' % lr).rstrip('0').rstrip('.'))+'_replace'+str(replace)+'_HL'+str(hl)\n",
    "        \n",
    "    # Print information if using night benchmark\n",
    "    if n_BM:\n",
    "        if development:\n",
    "            print('Night benchmark on development set')\n",
    "        else:\n",
    "            print('Night benchmark on test set')\n",
    "    # Print information if using uncontrolled charging\n",
    "    if uncontrolled:\n",
    "        if development:\n",
    "            print('Uncontrolled benchmark on development set')\n",
    "        else:\n",
    "            print('Uncontrolled benchmark on test set')\n",
    "\n",
    "    # Create the RL agent with a DQN\n",
    "    agent = DQNAgent(gamma=gamma, fc1_dims= hl[0], fc2_dims= hl[1], epsilon=1.0, lr=lr,\n",
    "                     input_dims=input_dims, n_actions=len(env.action_space), mem_size=100000, \n",
    "                     eps_min=0.1,batch_size=32, replace=replace, eps_dec=1e-5, optimizer=optimizer,\n",
    "                     chkpt_dir=store_dir,algo='DQNAgent', env_name='ChargingEnv-v0')\n",
    "        \n",
    "    # Load agent/model parameters from previously trained model\n",
    "    if load_checkpoint:\n",
    "        if test_final:\n",
    "            agent.load_models_final()\n",
    "            filename = filename + '_finalmodel'\n",
    "        elif mlp:\n",
    "            sup_model = SupModel().load_model_mlp(store_dir, mlp_hl, mlp_af, mlp_sl)\n",
    "            sup_scaler = SupModel().load_scaler(store_dir)\n",
    "            filename = 'MLP_'\n",
    "        elif knn:\n",
    "            sup_model = SupModel().load_model_kneighbors(store_dir, k)\n",
    "            sup_scaler = SupModel().load_scaler(store_dir)\n",
    "            filename = 'KNN_'\n",
    "        elif n_BM:\n",
    "            filename = dataset + '_BENCHMARK_Night_2-6'\n",
    "        elif uncontrolled:\n",
    "            filename = dataset + '_BENCHMARK_Uncontrolled'\n",
    "        else:\n",
    "            agent.load_models()\n",
    "        # Do not take any random actions, strictly act according to policy\n",
    "        agent.epsilon = 0\n",
    "    \n",
    "    #n_steps = 0\n",
    "    \n",
    "    # Lists to store all relevant data while training or testing\n",
    "    price_list, soc_list, action_list, dates, day_cats, starts, ends, scores, avg_scores, eps_history, pen_history, steps_array, final_soc, \\\n",
    "    discounted_action_list, temp_list = [], [], [], [], [], [], [], [], [], [], [], [], [], [], []\n",
    "    \n",
    "    for i in range(n_episodes):\n",
    "        # Get initial observation from the environment\n",
    "        observation = env.reset(test, i)\n",
    "        # Here, the return is called 'score'\n",
    "        score = 0\n",
    "        # Create lists to store training/test data\n",
    "        episode_prices, episode_soc, episode_actions, episode_day_cats, episode_discounted_actions, episode_temps = [], [], [], [], [], []\n",
    "        \n",
    "        # Loop for 24 h/steps in each episode/game\n",
    "        for n_steps in range(24):\n",
    "            \n",
    "            # No action can be taken while vehicle is not parked\n",
    "            if env.parking == 0:\n",
    "                action = '-'\n",
    "                # Store each action taken\n",
    "                episode_actions.append(action)\n",
    "                episode_discounted_actions.append(action)\n",
    "                # Receive new observation and reward(=0)\n",
    "                observation_, reward = env.non_parking_step()\n",
    "                score += reward\n",
    "                \n",
    "            # While the vehicle is parked choose action according to current policy or benchmark strategy\n",
    "            else:\n",
    "                # Test with night benchmarking approach: immediately discharge the vehicle in the evening, only charge between 02:00-06:00\n",
    "                if n_BM:\n",
    "                    \n",
    "                    if n_steps<14 and env.soc != 0:\n",
    "                        action=1 # Discharge the vehicle before 02:00 a.m.\n",
    "                    elif n_steps<14 and env.soc == 0:\n",
    "                        action=2 # Do nothing if the vehicle is fully discharged before 2 a.m.\n",
    "                    elif n_steps<18 and env.soc != 1:\n",
    "                        action=0 # Charge the vehicle between 2-6 a.m.\n",
    "                    else:\n",
    "                        action=2 # Do nothing after vehicle is fully charged                \n",
    "                \n",
    "                # Test with simple benchmarking approach: always charge the vehilce (no control mechanism and V2G/V2H)\n",
    "                elif uncontrolled:\n",
    "                    \n",
    "                    action=0\n",
    "                \n",
    "                # test with supervised model\n",
    "                elif mlp or knn:\n",
    "                    obs = np.array(observation)\n",
    "                    # Scale data\n",
    "                    obs = sup_scaler.transform(obs.reshape(1, -1))\n",
    "                    # Predict optimal action based on observation\n",
    "                    action = sup_model.predict(obs.reshape(1, -1))[0]\n",
    "\n",
    "                # Deep reinforcement agent takes action according to policy learned\n",
    "                else:\n",
    "                    action = agent.choose_action(observation)\n",
    "\n",
    "                # Store each action taken for evaluation and visualization\n",
    "                episode_actions.append(env.action_space[action]/env.charging_rate)\n",
    "                episode_discounted_actions.append(env.discounted_action)\n",
    "                \n",
    "                # Take a step and receive reward and new observation\n",
    "                observation_, reward = env.step(action)\n",
    "                score += reward\n",
    "                \n",
    "                # Fill replay memory while training\n",
    "                if not load_checkpoint:                    \n",
    "                    agent.store_transition(observation, action,\n",
    "                                         reward, observation_)\n",
    "                    \n",
    "                    # Start learning after defined number of random steps\n",
    "                    if i > pre_training_steps:\n",
    "                        agent.learn()\n",
    "            \n",
    "            # Store all prices, temps, soc for each episode\n",
    "            episode_prices.append(env.hourly_prices['Spot'][n_steps+168])\n",
    "            episode_temps.append(env.hourly_prices['temp'][n_steps+168])\n",
    "            episode_soc.append(env.soc)\n",
    "            episode_day_cats.append(env.day_cat) if env.day_cat not in episode_day_cats else episode_day_cats\n",
    "            \n",
    "            # Update observation\n",
    "            observation = observation_\n",
    "        \n",
    "        # Store all relevant data for evaluation\n",
    "        temp_list.append(episode_temps)\n",
    "        price_list.append(episode_prices)\n",
    "        soc_list.append(episode_soc)\n",
    "        action_list.append(episode_actions)\n",
    "        discounted_action_list.append(episode_discounted_actions)\n",
    "        dates.append(env.game_date)\n",
    "        day_cats.append(episode_day_cats)\n",
    "        starts.append(env.start_time)\n",
    "        ends.append(env.end_time)\n",
    "        final_soc.append(env.soc)\n",
    "        scores.append(score)\n",
    "        eps_history.append(agent.epsilon)\n",
    "        pen_history.append(env.penalty_coefficient)\n",
    "        avg_score = np.mean(scores[-100:])\n",
    "        avg_scores.append(avg_score)\n",
    "        \n",
    "        # Print average score every 100 episodes\n",
    "        if i%100==0:\n",
    "            print('episode: ', i,'score: ', score,\n",
    "                 ' average score %.1f' % avg_score, 'best score %.2f' % best_score,\n",
    "                'final_soc', env.soc, 'epsilon %.2f' % agent.epsilon, 'steps', n_steps)\n",
    "        \n",
    "        # Store model parameters when new moving average score outperforms previous best model\n",
    "        if avg_score > best_score:\n",
    "            if not load_checkpoint:\n",
    "                agent.save_models()\n",
    "            best_score = avg_score\n",
    "\n",
    "    # Store final model parameters after all training episodes\n",
    "    if not load_checkpoint:\n",
    "        agent.save_models_final()\n",
    "    \n",
    "    # Calculate the average score\n",
    "    score = sum(scores)/len(scores)\n",
    "\n",
    "    print('The average score is ', sum(scores)/len(scores))\n",
    "\n",
    "    # Store data for all training episodes in csv file\n",
    "    if store_results:\n",
    "        DataHandler().store_results(price_list, soc_list, action_list, dates, \\\n",
    "                                    day_cats, starts, ends, scores, \\\n",
    "                                    avg_scores, final_soc, eps_history, pen_history, \\\n",
    "                                    filename, optimizer, gamma, lr, replace, store_dir, \\\n",
    "                                    discounted_action_list,temp_list)\n",
    "    else: \n",
    "        print('Store results disabled')\n",
    "    \n",
    "\n",
    "if __name__ == '__main__':\n",
    "    i = 1\n",
    "    cwd = os.getcwd()\n",
    "    \n",
    "    ''' SET INPUT FEATURES WITH STRING '''\n",
    "    # The obs string and the resepctive input features variables must have been defined in charging_env.py\n",
    "    obs = 'obs4(t_sin,t_cos,daycat,temp0)'\n",
    "    # Set respective input dimensions for the DQN\n",
    "    input_dims = 4\n",
    "    \n",
    "    ''' SET DQN AGENT PARAMETERS '''\n",
    "    # If you want to test an agent that has been trained already the parameters have to match!\n",
    "    optimizer = 'Adam'\n",
    "    gamma = 0.8\n",
    "    lr = 0.0001\n",
    "    replace = 2\n",
    "    hl = [64,64]\n",
    "    \n",
    "    ''' SELECT AT LEAST ONE OF THE THREE FOLLOWING OPTIONS '''\n",
    "    # Select dataset for training or test on test/dev set\n",
    "    do_train = False\n",
    "    do_dev = False\n",
    "    do_test = True\n",
    "    \n",
    "    ''' SELECT NO MORE THAN ONE OF THE FOUR FOLLOWING OPTIONS '''\n",
    "    # Select true if you want to test a previously trained k-NN classifer\n",
    "    knn = False\n",
    "    # Select true if you want to test a previously trained MLP classifer\n",
    "    mlp = False\n",
    "    # Select if you want to test the night benchmark\n",
    "    n_BM = False\n",
    "    # Select if you want to test uncontrolled charging\n",
    "    uncontrolled = False\n",
    "\n",
    "    ''' IF k-NN OR MLP SELECTED -> SPECIFY PARAMETERS '''    \n",
    "    # Specifiy k-nearest neighbors parameters if required\n",
    "    k = 15\n",
    "    # Specify MLP parameters if required\n",
    "    mlp_hl = (16)\n",
    "    mlp_af = 'relu'\n",
    "    mlp_sl = 'adam'\n",
    "    \n",
    "    ''' SELECT IF YOU WANT TO STORE TRAIN/TEST INFORMATION IN A CSV FILE '''\n",
    "    store_results = False\n",
    "    \n",
    "    \n",
    "    ''' NO ADJUSTMENTS REQUIRED FROM HERE ''' \n",
    "    # Set store directory depending on previous decision\n",
    "    if knn:\n",
    "        info = ' | k = ' + str(k) + ' | ' + obs\n",
    "        store_dir = cwd +'/knn_models/'+ 'KNeighbors_k(' + str(k) + ')' + '_' + obs\n",
    "    elif mlp:\n",
    "        info = ' | optimizer=' + mlp_sl + ' | activation function: ' + mlp_af + ' | hl: ' + str(mlp_hl) + ' | ' + obs\n",
    "        store_dir = cwd +'/mlp_models/' + 'MLP_hl(' + str(mlp_hl) + ')_af(' + str(mlp_af) + ')_sl(' + str(mlp_sl) + ')' + '_' + obs\n",
    "    elif n_BM or uncontrolled:\n",
    "        info = ' Rule-based benchmark specified above '\n",
    "        store_dir = cwd\n",
    "    else:\n",
    "        info = ' | optimizer=' + optimizer + ' | gamma=' + str(gamma) + ' | lr='+(('%.15f' % lr).rstrip('0').rstrip('.')) + ' | replace=' + str(replace) + ' | HL: ' + str(hl) + ' | ' + obs\n",
    "        store_dir = cwd +'/dqn_models/'+ optimizer+'_gamma'+str(gamma)+'_lr'+(('%.15f' % lr).rstrip('0').rstrip('.'))+'_replace'+str(replace)+'_HL'+str(hl) + '_' + obs\n",
    "    \n",
    "    # Train\n",
    "    if do_train:\n",
    "        print('---------- TRAIN session: ', i, info)\n",
    "        os.makedirs(store_dir)\n",
    "        start = time.time()            \n",
    "        run_model(optimizer, gamma, lr, replace, hl, k, store_dir, mlp_hl, mlp_af, mlp_sl, input_dims, mlp, knn, n_BM, uncontrolled, obs, store_results, development=False, test=False, test_final=False)\n",
    "        end = time.time()\n",
    "        print('Training took ', end-start, ' seconds...')\n",
    "    \n",
    "    # DEV\n",
    "    if do_dev:\n",
    "        print('---------- DEV session: ' +str(i)+info)\n",
    "        run_model(optimizer, gamma, lr, replace, hl, k, store_dir, mlp_hl, mlp_af, mlp_sl, input_dims, mlp, knn, n_BM, uncontrolled, obs, store_results, development=True, test=True, test_final=False)\n",
    "\n",
    "    # Test\n",
    "    if do_test:\n",
    "        print('---------- TEST session: ', i, info)\n",
    "        run_model(optimizer, gamma, lr, replace, hl, k, store_dir, mlp_hl, mlp_af, mlp_sl, input_dims, mlp, knn, n_BM, uncontrolled, obs, store_results, development=False, test=True, test_final=False)\n",
    "    \n",
    "    # Print information that nothing is trained or tested...\n",
    "    if not (do_train or do_dev or do_test):\n",
    "        print('Select do_train / do_dev / do_test')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}