part_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "df1e3ab9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import cvxpy as cp\n",
    "import numpy as np\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8f9e6dc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "POS = [ \"W\",\"N\",\"E\",\"S\",\"C\" ]    # position of IJ\n",
    "MAT = [ 0,1,2 ]    # material with IJ\n",
    "ARROW = [ 0,1,2,3 ]    # number of arrows\n",
    "STATE = [ \"D\",\"R\" ]    # ready and dormant state of MM\n",
    "HEALTH = [ 0,25,50,75,100 ]    # MM’s health\n",
    "ACTION = [ \"UP\",\"LEFT\",\"DOWN\",\"RIGHT\",\"STAY\",\"SHOOT\",\"HIT\",\"CRAFT\",\"GATHER\",\"NONE\" ]    # POSSIBLE ACTIONS OF IJ\n",
    "\n",
    "TEAM = 25\n",
    "\n",
    "ARR = [ 0.5, 1, 2 ]\n",
    "Y = ARR[TEAM % 3]\n",
    "STEPCOST = -10/Y\n",
    "\n",
    "GAMMA = 0.999\n",
    "DELTA = 0.001\n",
    "\n",
    "BLADEHITDAMAGE = 50\n",
    "ARROWHITDAMAGE = 25\n",
    "\n",
    "NEGATEREWARD = 40\n",
    "os.makedirs('outputs', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "54b5388a",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_state = []\n",
    "for pos in POS:\n",
    "    for mat in MAT:\n",
    "        for arrow in ARROW:\n",
    "            for state in STATE:\n",
    "                for health in HEALTH:\n",
    "                    all_state.append((pos,mat,arrow,state,health))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1b39359a",
   "metadata": {},
   "outputs": [],
   "source": [
    "state_actions = {} # state actions in dict.\n",
    "for s in all_state:\n",
    "    if s[-1]==0:\n",
    "        state_actions[s] = ['NONE']\n",
    "    elif s[0]=='W':\n",
    "        state_actions[s] = [ 'RIGHT','STAY']\n",
    "        if s[2]!=0:\n",
    "            state_actions[s].append('SHOOT')\n",
    "    elif s[0]=='N':\n",
    "        state_actions[s] = [ 'DOWN','STAY']\n",
    "        if (s[1]>0):\n",
    "                state_actions[s].append('CRAFT')\n",
    "    elif s[0]=='E':\n",
    "        state_actions[s] = [ 'LEFT','STAY' ]\n",
    "        if s[2]!=0:\n",
    "            state_actions[s].append('SHOOT')\n",
    "        state_actions[s].append('HIT')\n",
    "    elif s[0]=='S':\n",
    "        state_actions[s] = [ 'UP','STAY','GATHER' ]\n",
    "    else:\n",
    "        state_actions[s] = [ 'UP','LEFT','DOWN','RIGHT','STAY' ]\n",
    "        if s[2]>0:\n",
    "            state_actions[s].append('SHOOT')\n",
    "        state_actions[s].append('HIT')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0a2e7dfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pfsb(current_state, action):\n",
    "    pfsb = []\n",
    "    (pos,mat,arrow,state,health) = current_state\n",
    "    if state=='D':\n",
    "        \n",
    "        if action == 'UP':\n",
    "            if pos=='C':\n",
    "                pfsb.append(('N',mat,arrow,'D',health,0.85*0.8))\n",
    "                pfsb.append(('N',mat,arrow,'R',health,0.85*0.2))\n",
    "            else:\n",
    "                pfsb.append(('C',mat,arrow,'D',health,0.85*0.8))\n",
    "                pfsb.append(('C',mat,arrow,'R',health,0.85*0.2))\n",
    "            pfsb.append(('E',mat,arrow,'D',health,0.15*0.8))\n",
    "            pfsb.append(('E',mat,arrow,'R',health,0.15*0.2))\n",
    "            \n",
    "        elif action == 'DOWN':\n",
    "            if pos=='C':\n",
    "                pfsb.append(('S',mat,arrow,'D',health,0.85*0.8))\n",
    "                pfsb.append(('S',mat,arrow,'R',health,0.85*0.2))\n",
    "            else:\n",
    "                pfsb.append(('C',mat,arrow,'D',health,0.85*0.8))\n",
    "                pfsb.append(('C',mat,arrow,'R',health,0.85*0.2))\n",
    "            pfsb.append(('E',mat,arrow,'D',health,0.15*0.8))\n",
    "            pfsb.append(('E',mat,arrow,'R',health,0.15*0.2))\n",
    "            \n",
    "        elif action == 'LEFT':\n",
    "            if pos=='C':\n",
    "                pfsb.append(('W',mat,arrow,'D',health,0.85*0.8))\n",
    "                pfsb.append(('W',mat,arrow,'R',health,0.85*0.2))\n",
    "                pfsb.append(('E',mat,arrow,'D',health,0.15*0.8))\n",
    "                pfsb.append(('E',mat,arrow,'R',health,0.15*0.2))\n",
    "            else:\n",
    "                pfsb.append(('C',mat,arrow,'D',health,1*0.8))\n",
    "                pfsb.append(('C',mat,arrow,'R',health,1*0.2))\n",
    "        \n",
    "        elif action == 'RIGHT':\n",
    "            if pos=='C':\n",
    "                pfsb.append(('E',mat,arrow,'D',health,1*0.8))\n",
    "                pfsb.append(('E',mat,arrow,'R',health,1*0.2))\n",
    "            else:\n",
    "                pfsb.append(('C',mat,arrow,'D',health,1*0.8))\n",
    "                pfsb.append(('C',mat,arrow,'R',health,1*0.2))\n",
    "        \n",
    "        elif action == 'STAY':\n",
    "            if pos in ['C','S','N']:\n",
    "                pfsb.append((pos,mat,arrow,'R',health,0.85*0.2))\n",
    "                pfsb.append((pos,mat,arrow,'D',health,0.85*0.8))\n",
    "                pfsb.append(('E',mat,arrow,'R',health,0.15*0.2))\n",
    "                pfsb.append(('E',mat,arrow,'D',health,0.15*0.8))\n",
    "            else:\n",
    "                pfsb.append((pos,mat,arrow,'R',health,1*0.2))\n",
    "                pfsb.append((pos,mat,arrow,'D',health,1*0.8))\n",
    "                \n",
    "        elif action == 'HIT':\n",
    "            if pos=='C':\n",
    "                pfsb.append((pos,mat,arrow,'D',max(health-BLADEHITDAMAGE,0),0.1*0.8))\n",
    "                pfsb.append((pos,mat,arrow,'D',health,0.9*0.8))\n",
    "                pfsb.append((pos,mat,arrow,'R',max(health-BLADEHITDAMAGE,0),0.1*0.2))\n",
    "                pfsb.append((pos,mat,arrow,'R',health,0.9*0.2))\n",
    "            else:\n",
    "                pfsb.append((pos,mat,arrow,'D',max(health-BLADEHITDAMAGE,0),0.2*0.8))\n",
    "                pfsb.append((pos,mat,arrow,'D',health,0.8*0.8))\n",
    "                pfsb.append((pos,mat,arrow,'R',max(health-BLADEHITDAMAGE,0),0.2*0.2))\n",
    "                pfsb.append((pos,mat,arrow,'R',health,0.8*0.2))\n",
    "\n",
    "        elif action == 'SHOOT':\n",
    "            if arrow==0:\n",
    "                pfsb.append((pos,mat,arrow,'D',health,1*0.8))\n",
    "                pfsb.append((pos,mat,arrow,'R',health,1*0.2))\n",
    "            else:\n",
    "                if pos=='W':\n",
    "                    pfsb.append((pos,mat,arrow-1,'D',max(health-ARROWHITDAMAGE,0),0.25*0.8))\n",
    "                    pfsb.append((pos,mat,arrow-1,'R',max(health-ARROWHITDAMAGE,0),0.25*0.2))\n",
    "                    pfsb.append((pos,mat,arrow-1,'D',health,0.75*0.8))\n",
    "                    pfsb.append((pos,mat,arrow-1,'R',health,0.75*0.2))\n",
    "                elif pos=='C':\n",
    "                    pfsb.append((pos,mat,arrow-1,'D',max(health-ARROWHITDAMAGE,0),0.5*0.8))\n",
    "                    pfsb.append((pos,mat,arrow-1,'R',max(health-ARROWHITDAMAGE,0),0.5*0.2))\n",
    "                    pfsb.append((pos,mat,arrow-1,'D',health,0.5*0.8))\n",
    "                    pfsb.append((pos,mat,arrow-1,'R',health,0.5*0.2))\n",
    "                else:\n",
    "                    pfsb.append((pos,mat,arrow-1,'D',max(health-ARROWHITDAMAGE,0),0.9*0.8))\n",
    "                    pfsb.append((pos,mat,arrow-1,'R',max(health-ARROWHITDAMAGE,0),0.9*0.2))\n",
    "                    pfsb.append((pos,mat,arrow-1,'D',health,0.1*0.8))\n",
    "                    pfsb.append((pos,mat,arrow-1,'R',health,0.1*0.2))\n",
    "        \n",
    "        elif action == 'GATHER':\n",
    "            if mat<2:\n",
    "                pfsb.append((pos,mat+1,arrow,'D',health,0.75*0.8))\n",
    "                pfsb.append((pos,mat+1,arrow,'R',health,0.75*0.2))\n",
    "                pfsb.append((pos,mat,arrow,'D',health,0.25*0.8))\n",
    "                pfsb.append((pos,mat,arrow,'R',health,0.25*0.2))\n",
    "            else:\n",
    "                pfsb.append((pos,mat,arrow,'D',health,1*0.8))\n",
    "                pfsb.append((pos,mat,arrow,'R',health,1*0.2))\n",
    "        \n",
    "        elif action == 'CRAFT':\n",
    "            if (mat>0):\n",
    "                if arrow==0:\n",
    "                    pfsb.append((pos,mat-1,1,'D',health,0.5*0.8))\n",
    "                    pfsb.append((pos,mat-1,2,'D',health,0.35*0.8))\n",
    "                    pfsb.append((pos,mat-1,3,'D',health,0.15*0.8))\n",
    "                    pfsb.append((pos,mat-1,1,'R',health,0.5*0.2))\n",
    "                    pfsb.append((pos,mat-1,2,'R',health,0.35*0.2))\n",
    "                    pfsb.append((pos,mat-1,3,'R',health,0.15*0.2))\n",
    "                elif arrow==1:\n",
    "                    pfsb.append((pos,mat-1,2,'D',health,0.5*0.8))\n",
    "                    pfsb.append((pos,mat-1,3,'D',health,0.5*0.8))\n",
    "                    pfsb.append((pos,mat-1,2,'R',health,0.5*0.2))\n",
    "                    pfsb.append((pos,mat-1,3,'R',health,0.5*0.2))\n",
    "                elif arrow in [2,3]:\n",
    "                    pfsb.append((pos,mat-1,3,'R',health,1*0.2))\n",
    "                    pfsb.append((pos,mat-1,3,'D',health,1*0.8))\n",
    "            else:\n",
    "                pfsb.append((pos,mat,arrow,'D',health,1*0.8))\n",
    "                pfsb.append((pos,mat,arrow,'R',health,1*0.2))\n",
    "                    \n",
    "    else:\n",
    "        \n",
    "        if action == 'UP':\n",
    "            if pos=='C':\n",
    "                pfsb.append((pos,mat,0,'D',min(health+25,100),0.5))\n",
    "                pfsb.append(('N',mat,arrow,'R',health,0.85*0.5))\n",
    "                pfsb.append(('E',mat,arrow,'R',health,0.15*0.5))\n",
    "            else:\n",
    "                pfsb.append(('C',mat,arrow,'D',health,0.5))\n",
    "                pfsb.append(('C',mat,arrow,'R',health,0.85*0.5))\n",
    "                pfsb.append(('E',mat,arrow,'R',health,0.15*0.5))\n",
    "\n",
    "        elif action == 'DOWN':\n",
    "            if pos=='C':\n",
    "                pfsb.append((pos,mat,0,'D',min(health+25,100),0.5))\n",
    "                pfsb.append(('S',mat,arrow,'R',health,0.85*0.5))\n",
    "                pfsb.append(('E',mat,arrow,'R',health,0.15*0.5))\n",
    "            else:\n",
    "                pfsb.append(('C',mat,arrow,'R',health,0.85*0.5))\n",
    "                pfsb.append(('E',mat,arrow,'R',health,0.15*0.5))\n",
    "                pfsb.append(('C',mat,arrow,'D',health,0.85*0.5))\n",
    "                pfsb.append(('E',mat,arrow,'D',health,0.15*0.5))\n",
    "            \n",
    "        elif action == 'LEFT':\n",
    "            pfsb.append((pos,mat,0,'D',min(health+25,100),0.5))\n",
    "            if pos=='C':\n",
    "                pfsb.append(('W',mat,arrow,'R',health,0.85*0.5))\n",
    "                pfsb.append(('E',mat,arrow,'R',health,0.15*0.5))\n",
    "            else:\n",
    "                pfsb.append(('C',mat,arrow,'R',health,1*0.5))\n",
    "        \n",
    "        elif action == 'RIGHT':\n",
    "            if pos=='C':\n",
    "                pfsb.append((pos,mat,0,'D',min(health+25,100),0.5))\n",
    "                pfsb.append(('E',mat,arrow,'R',health,1*0.5))\n",
    "            else:\n",
    "                pfsb.append(('C',mat,arrow,'R',health,1*0.5))\n",
    "                pfsb.append(('C',mat,arrow,'D',health,1*0.5))\n",
    "\n",
    "        elif action == 'STAY':\n",
    "            if pos=='C':\n",
    "                pfsb.append((pos,mat,0,'D',min(health+25,100),0.5))\n",
    "                pfsb.append((pos,mat,arrow,'R',health,0.85*0.5))\n",
    "                pfsb.append(('E',mat,arrow,'R',health,0.15*0.5))\n",
    "            \n",
    "            elif pos=='E':\n",
    "                pfsb.append((pos,mat,0,'D',min(health+25,100),0.5))\n",
    "                pfsb.append(('E',mat,arrow,'R',health,1*0.5))\n",
    "                \n",
    "            elif pos in ['S','N']:\n",
    "                pfsb.append((pos,mat,arrow,'R',health,0.85*0.5))\n",
    "                pfsb.append(('E',mat,arrow,'R',health,0.15*0.5))\n",
    "                pfsb.append((pos,mat,arrow,'D',health,0.85*0.5))\n",
    "                pfsb.append(('E',mat,arrow,'D',health,0.15*0.5))\n",
    "                \n",
    "            else:\n",
    "                pfsb.append((pos,mat,arrow,'R',health,1*0.5))\n",
    "                pfsb.append((pos,mat,arrow,'D',health,1*0.5))\n",
    "                \n",
    "        elif action == 'HIT':\n",
    "            pfsb.append((pos,mat,0,'D',min(health+25,100),0.5))\n",
    "            if pos=='C':\n",
    "                pfsb.append((pos,mat,arrow,'R',max(health-BLADEHITDAMAGE,0),0.1*0.5))\n",
    "                pfsb.append((pos,mat,arrow,'R',health,0.9*0.5))\n",
    "            else:\n",
    "                pfsb.append((pos,mat,arrow,'R',max(health-BLADEHITDAMAGE,0),0.2*0.5))\n",
    "                pfsb.append((pos,mat,arrow,'R',health,0.8*0.5))\n",
    "\n",
    "        elif action == 'SHOOT':\n",
    "            if arrow==0:\n",
    "                pfsb.append((pos,mat,arrow,'R',health,1*0.5))\n",
    "                if pos=='W':\n",
    "                    pfsb.append((pos,mat,arrow,'D',health,1*0.5))\n",
    "                else:\n",
    "                    pfsb.append((pos,mat,0,'D',min(health+25,100),1*0.5))\n",
    "            else:\n",
    "                if pos=='W':\n",
    "                    pfsb.append((pos,mat,arrow-1,'R',max(health-ARROWHITDAMAGE,0),0.25*0.5))\n",
    "                    pfsb.append((pos,mat,arrow-1,'R',health,0.75*0.5))\n",
    "                    pfsb.append((pos,mat,arrow-1,'D',max(health-ARROWHITDAMAGE,0),0.25*0.5))\n",
    "                    pfsb.append((pos,mat,arrow-1,'D',health,0.75*0.5))\n",
    "                elif pos=='C':\n",
    "                    pfsb.append((pos,mat,0,'D',min(health+25,100),0.5))\n",
    "                    pfsb.append((pos,mat,arrow-1,'R',max(health-ARROWHITDAMAGE,0),0.5*0.5))\n",
    "                    pfsb.append((pos,mat,arrow-1,'R',health,0.5*0.5))\n",
    "                else:\n",
    "                    pfsb.append((pos,mat,0,'D',min(health+25,100),0.5))\n",
    "                    pfsb.append((pos,mat,arrow-1,'R',max(health-ARROWHITDAMAGE,0),0.9*0.5))\n",
    "                    pfsb.append((pos,mat,arrow-1,'R',health,0.1*0.5))\n",
    "        \n",
    "        elif action == 'GATHER':\n",
    "            if mat<2:\n",
    "                pfsb.append((pos,mat+1,arrow,'R',health,0.75*0.5))\n",
    "                pfsb.append((pos,mat,arrow,'R',health,0.25*0.5))\n",
    "                pfsb.append((pos,mat+1,arrow,'D',health,0.75*0.5))\n",
    "                pfsb.append((pos,mat,arrow,'D',health,0.25*0.5))\n",
    "            else:\n",
    "                pfsb.append((pos,mat,arrow,'R',health,1*0.5))\n",
    "                pfsb.append((pos,mat,arrow,'D',health,1*0.5))\n",
    "\n",
    "        elif action == 'CRAFT':\n",
    "            if (mat>0):\n",
    "                if arrow==0:\n",
    "                    pfsb.append((pos,mat-1,1,'R',health,0.5*0.5))\n",
    "                    pfsb.append((pos,mat-1,2,'R',health,0.35*0.5))\n",
    "                    pfsb.append((pos,mat-1,3,'R',health,0.15*0.5))\n",
    "                    pfsb.append((pos,mat-1,1,'D',health,0.5*0.5))\n",
    "                    pfsb.append((pos,mat-1,2,'D',health,0.35*0.5))\n",
    "                    pfsb.append((pos,mat-1,3,'D',health,0.15*0.5))\n",
    "                elif arrow==1:\n",
    "                    pfsb.append((pos,mat-1,2,'R',health,0.5*0.5))\n",
    "                    pfsb.append((pos,mat-1,3,'R',health,0.5*0.5))\n",
    "                    pfsb.append((pos,mat-1,2,'D',health,0.5*0.5))\n",
    "                    pfsb.append((pos,mat-1,3,'D',health,0.5*0.5))\n",
    "                elif arrow in [2,3]:\n",
    "                    pfsb.append((pos,mat-1,3,'D',health,1*0.5))\n",
    "                    pfsb.append((pos,mat-1,3,'R',health,1*0.5))\n",
    "            else:\n",
    "                pfsb.append((pos,mat,arrow,'R',health,1*0.5))\n",
    "                pfsb.append((pos,mat,arrow,'D',health,1*0.5))\n",
    "    return pfsb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fbee7dcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_file(filename):\n",
    "    with open(filename,'r') as read_file:\n",
    "        data=json.load(read_file)\n",
    "    return data\n",
    "\n",
    "def write_file(filename,data):\n",
    "    with open(filename,'w') as write_file:\n",
    "        json.dump(data, write_file, indent = 4)\n",
    "    return read_file(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "59fd6c35",
   "metadata": {},
   "outputs": [],
   "source": [
    "R = []\n",
    "for s in all_state:\n",
    "    for a in state_actions[s]:\n",
    "        expected_reward = 0\n",
    "        pfsp = get_pfsb(s, a)\n",
    "        for state_prob in pfsp:\n",
    "            reward = STEPCOST\n",
    "            (pos,mat,arrow,state,health,prob) = state_prob\n",
    "            if((pos==s[0]) and\n",
    "               (pos in ['E', 'C']) and\n",
    "              (mat==s[1]) and\n",
    "              (arrow==0) and\n",
    "              (state=='D' and s[3]=='R') and\n",
    "              ((health==s[4]+25) or ((health==100)and(s[4]==100)))\n",
    "             ):\n",
    "                reward -= NEGATEREWARD\n",
    "            expected_reward += (reward*prob)\n",
    "        R.append(expected_reward)\n",
    "        \n",
    "R = np.array(R, dtype=float).reshape(1,len(R))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0ad057cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "A = []\n",
    "\n",
    "r_state_map = {}\n",
    "for i in range(len(all_state)):\n",
    "    r_state_map[all_state[i]]=i\n",
    "\n",
    "for s1 in all_state:\n",
    "    row = []\n",
    "    for s2 in all_state:\n",
    "        c_cluster = []\n",
    "        for a in state_actions[s2]:\n",
    "            c_cluster.append(0)\n",
    "        row.append(c_cluster)\n",
    "    A.append(row)\n",
    "\n",
    "for s in all_state:\n",
    "    for ai in range(len(state_actions[s])):\n",
    "        j = r_state_map[s]\n",
    "        k = ai\n",
    "        if s[-1]==0:\n",
    "            A[j][j][k]=1\n",
    "            continue\n",
    "        \n",
    "        a = state_actions[s][ai]\n",
    "        pfsp = get_pfsb(s, a)\n",
    "        for state_prob in pfsp:\n",
    "            (pos,mat,arrow,state,health,prob) = state_prob\n",
    "            pfs = (pos,mat,arrow,state,health)\n",
    "            i = r_state_map[pfs]\n",
    "            A[j][j][k] += prob\n",
    "            A[i][j][k] -= prob\n",
    "\n",
    "LP_A = []\n",
    "for i in range(len(A)):\n",
    "    row = []\n",
    "    for cluster in A[i]:\n",
    "        for p in cluster:\n",
    "            row.append(p)\n",
    "    row = np.array(row)\n",
    "    LP_A.append(row)\n",
    "LP_A = np.array(LP_A, dtype=float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2d7205af",
   "metadata": {},
   "outputs": [],
   "source": [
    "ALPHA = np.zeros(len(all_state))\n",
    "ALPHA[r_state_map[('C',2,3,'R',100)]]=1\n",
    "ALPHA = np.array(ALPHA, dtype=float).reshape(len(all_state),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c071bede",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = cp.Variable(shape=(len(R[0]), 1), name=\"x\")\n",
    "constraints = [cp.matmul(LP_A,x) == ALPHA, x>=0]\n",
    "objective = cp.Maximize(cp.matmul(R,x))\n",
    "problem = cp.Problem(objective, constraints)\n",
    "solution = problem.solve()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7476d234",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimal_policy = {}\n",
    "cum_i = 0\n",
    "for i in range(len(all_state)):\n",
    "    s = all_state[i]\n",
    "    max_reward = -1000000\n",
    "    for j in range(len(state_actions[s])):\n",
    "        a = state_actions[s][j]\n",
    "        if max_reward <= R[0][cum_i+j]:\n",
    "            max_reward = R[0][cum_i+j]\n",
    "            optimal_policy[s] = a\n",
    "    cum_i += len(state_actions[s])      \n",
    "\n",
    "policy = []\n",
    "for s in optimal_policy:\n",
    "    (pos,mat,arrow,state,health) = s\n",
    "    sublist = []\n",
    "    state = [pos[0],mat,arrow,state,health]\n",
    "    action = optimal_policy[s]\n",
    "    sublist.append(state)\n",
    "    sublist.append(action)\n",
    "    policy.append(sublist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f6277dce",
   "metadata": {},
   "outputs": [],
   "source": [
    "dump_A = LP_A.tolist()\n",
    "dump_R = R[0][:].tolist()\n",
    "dump_ALPHA = ALPHA[:,0].tolist()\n",
    "dump_X = list(x.value.flatten())\n",
    "dump_objective = solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "980dc15c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dump_dict =  {\n",
    "    \"a\": dump_A,\n",
    "    \"r\": dump_R,\n",
    "    \"alpha\": dump_ALPHA,\n",
    "    \"x\": dump_X,\n",
    "    \"policy\": policy,\n",
    "    \"objective\": dump_objective\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f13764ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(np.array(dump_A))\n",
    "# print(np.array(dump_R))\n",
    "# print(np.array(dump_ALPHA))\n",
    "# print(np.array(policy))\n",
    "# print(np.array(dump_objective))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7c162acf",
   "metadata": {},
   "outputs": [],
   "source": [
    "load_data = write_file(\"outputs/part_3_output.json\",dump_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb376596",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}