Fase1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random as random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "def obtainRMatrix(m):\n",
    "    Rsize = size**2\n",
    "    rMatrix = inicializeRMatrix(num_col,num_fil)\n",
    "    res=[]\n",
    "    for i in range(num_fil):\n",
    "        for j in range(num_col):\n",
    "            n0 = m[i][j]\n",
    "            if 0<i<num_fil-1:\n",
    "                rMatrix[m[i][j]][m[i-1][j]]=0\n",
    "                rMatrix[m[i][j]][m[i+1][j]]=0\n",
    "                \n",
    "                if(0<j<num_col-1):\n",
    "                    rMatrix[m[i][j]][m[i-1][j-1]]=0\n",
    "                    rMatrix[m[i][j]][m[i][j-1]]=0\n",
    "                    rMatrix[m[i][j]][m[i-1][j+1]]=0\n",
    "                    rMatrix[m[i][j]][m[i+1][j-1]]=0\n",
    "                    rMatrix[m[i][j]][m[i+1][j+1]]=0\n",
    "                    rMatrix[m[i][j]][m[i][j+1]]=0\n",
    "                    \n",
    "                if(j==0):\n",
    "                    rMatrix[m[i][j]][m[i-1][j+1]]=0\n",
    "                    rMatrix[m[i][j]][m[i][j+1]]=0\n",
    "                    rMatrix[m[i][j]][m[i+1][j+1]]=0\n",
    "                    \n",
    "                if(j==num_col-1):\n",
    "                    rMatrix[m[i][j]][m[i-1][j-1]]=0\n",
    "                    rMatrix[m[i][j]][m[i][j-1]]=0\n",
    "                    rMatrix[m[i][j]][m[i+1][j-1]]=0  \n",
    "                    \n",
    "            elif(i==0):\n",
    "                rMatrix[m[i][j]][m[i+1][j]]=0             \n",
    "                \n",
    "                if(j==0):\n",
    "                    rMatrix[m[i][j]][m[i+1][j+1]]=0             \n",
    "                    rMatrix[m[i][j]][m[i][j+1]]=0   \n",
    "                    \n",
    "                if(j==num_col-1):\n",
    "                    rMatrix[m[i][j]][m[i+1][j-1]]=0  \n",
    "                    rMatrix[m[i][j]][m[i][j-1]]=0 \n",
    "                    \n",
    "                if(0<j<num_col-1):\n",
    "                    rMatrix[m[i][j]][m[i+1][j-1]]=0  \n",
    "                    rMatrix[m[i][j]][m[i+1][j+1]]=0  \n",
    "                    rMatrix[m[i][j]][m[i][j+1]]=0  \n",
    "                    rMatrix[m[i][j]][m[i][j-1]]=0  \n",
    "                    \n",
    "            elif i==num_fil-1:\n",
    "                rMatrix[m[i][j]][m[i-1][j]]=0  \n",
    "                if(j==0):\n",
    "                    rMatrix[m[i][j]][m[i-1][j+1]]=0  \n",
    "                    rMatrix[m[i][j]][m[i][j+1]]=0  \n",
    "                if(j==num_col-1):\n",
    "                    rMatrix[m[i][j]][m[i-1][j-1]]=0  \n",
    "                    rMatrix[m[i][j]][m[i][j-1]]=0  \n",
    "                if(0<j<num_col-1):\n",
    "                    rMatrix[m[i][j]][m[i-1][j+1]]=0  \n",
    "                    rMatrix[m[i][j]][m[i-1][j-1]]=0  \n",
    "                    rMatrix[m[i][j]][m[i][j+1]]=0  \n",
    "                    rMatrix[m[i][j]][m[i][j-1]]=0  \n",
    "    return rMatrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addObjetives(R,objetives):\n",
    "    for objetive in objetives:\n",
    "        for i in range(num_fil**2):\n",
    "            R[i][objetive] = 100 if (R[i][objetive]==0) else -1       \n",
    "        R[objetive][objetive] = 100\n",
    "    return R          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.size(rMatrix[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addLayerToMatrix(matrix, layer, count):\n",
    "    for i in range(layer):\n",
    "        \n",
    "        matrix[layer][i] = int(count)\n",
    "        count=count+1\n",
    "        \n",
    "    for i in range(layer+1):\n",
    "        \n",
    "        matrix[layer-i][layer] = int(count)\n",
    "        count=count+1\n",
    "        \n",
    "    return [matrix, count]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createMatrix(m,tam):\n",
    "    layer=0\n",
    "    for i in range(tam):\n",
    "        [m,layer]=addLayerToMatrix(m,i,layer)\n",
    "    return m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inicializeRMatrix(numRows, numCols):\n",
    "    return np.array([-1 for _ in range((numRows**2)*(numCols**2))]).reshape(numRows**2, numCols**2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "¿Cuál es el tamaño del problema?  3\n"
     ]
    }
   ],
   "source": [
    "size = int(input('¿Cuál es el tamaño del problema? '))\n",
    "num_col = size\n",
    "num_fil = size\n",
    "states = (num_col*num_fil)**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "¿Cuál es el estado objetivo? 6\n"
     ]
    }
   ],
   "source": [
    "objetivo = int(input('¿Cuál es el estado objetivo?'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.zeros((num_fil,num_col))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0],\n",
       "       [0, 0, 0],\n",
       "       [0, 0, 0]])"
      ]
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 3, 8],\n",
       "       [1, 2, 7],\n",
       "       [4, 5, 6]])"
      ]
     },
     "execution_count": 356,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = createMatrix(a,size).astype(int)\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 3 8]\n",
      " [1 2 7]\n",
      " [4 5 6]]\n"
     ]
    }
   ],
   "source": [
    "print(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 1\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "rMatrix=obtainRMatrix(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1,  0,  0,  0, -1, -1, -1, -1, -1],\n",
       "       [ 0, -1,  0,  0,  0,  0, -1, -1, -1],\n",
       "       [ 0,  0, -1,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0, -1, -1, -1, -1,  0,  0],\n",
       "       [-1,  0,  0, -1, -1,  0, -1, -1, -1],\n",
       "       [-1,  0,  0, -1,  0, -1,  0,  0, -1],\n",
       "       [-1, -1,  0, -1, -1,  0, -1,  0, -1],\n",
       "       [-1, -1,  0,  0, -1,  0,  0, -1,  0],\n",
       "       [-1, -1,  0,  0, -1, -1, -1,  0, -1]])"
      ]
     },
     "execution_count": 359,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rMatrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -1,   0,   0,   0,  -1,  -1,  -1,  -1,  -1],\n",
       "       [  0,  -1,   0,   0,   0,   0,  -1,  -1,  -1],\n",
       "       [  0,   0,  -1,   0,   0,   0, 100,   0,   0],\n",
       "       [  0,   0,   0,  -1,  -1,  -1,  -1,   0,   0],\n",
       "       [ -1,   0,   0,  -1,  -1,   0,  -1,  -1,  -1],\n",
       "       [ -1,   0,   0,  -1,   0,  -1, 100,   0,  -1],\n",
       "       [ -1,  -1,   0,  -1,  -1,   0, 100,   0,  -1],\n",
       "       [ -1,  -1,   0,   0,  -1,   0, 100,  -1,   0],\n",
       "       [ -1,  -1,   0,   0,  -1,  -1,  -1,   0,  -1]])"
      ]
     },
     "execution_count": 360,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rMatrix=addObjetives(rMatrix,[objetivo])\n",
    "rMatrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "qMatrix = np.zeros((size**2,size**2)).astype(int)\n",
    "print(qMatrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Algoritmo de aprendizaje \n",
    "-----------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "def acciones ( matriz, estado ):\n",
    "    fila_estado = matriz [ estado , : ]\n",
    "    acciones = [ ]\n",
    "    for n in range ( len ( fila_estado ) ):\n",
    "        if (fila_estado [ n ] != -1):\n",
    "            acciones.append(n)\n",
    "    return (acciones)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "def maxQValue(Q, R, estado):\n",
    "    #obtener los estados accesibles desde la matriz R\n",
    "    rRow = R[estado]\n",
    "    qRow = Q[estado]\n",
    "    possibleStatesIndex = np.where(rRow != -1)\n",
    "    #obtener los valores en la matriz Q\n",
    "    qLista=qRow[possibleStatesIndex]\n",
    "    #devolver el máximo\n",
    "    maxValue=np.amax(qLista)\n",
    "    return maxValue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [],
   "source": [
    "def entrenarQ ( Q, R, Gamma, objetivo, num_episodios ):\n",
    "    while num_episodios > 0:\n",
    "        estado = random.randint ( 0, num_fil**2 - 1 )\n",
    "        while ( estado != objetivo ):\n",
    "            #acciones = acciones ( R, estado )\n",
    "            accion_elegida = random.choice ( acciones ( R, estado ) )\n",
    "            siguiente_estado = accion_elegida\n",
    "            Q [ estado, accion_elegida ] = R [ estado, accion_elegida ] + Gamma * maxQValue(Q,R,siguiente_estado)\n",
    "            estado = siguiente_estado\n",
    "        num_episodios = num_episodios - 1\n",
    "    return Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500.0"
      ]
     },
     "execution_count": 347,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxQValue(qMatrix,rMatrix,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  0  64  80  64   0   0   0   0   0]\n",
      " [ 64   0  80  64  64  80   0   0   0]\n",
      " [ 64  64   0  64  64  80 100  80  64]\n",
      " [ 64  64  80   0   0   0   0  80  64]\n",
      " [  0  64  80   0   0  80   0   0   0]\n",
      " [  0  64  80   0  64   0 100  80   0]\n",
      " [  0   0   0   0   0   0   0   0   0]\n",
      " [  0   0  80  64   0  80 100   0  64]\n",
      " [  0   0  80  64   0   0   0  80   0]]\n"
     ]
    }
   ],
   "source": [
    "entrenarQ(qMatrix, rMatrix, 0.8, objetivo, 1000)\n",
    "\n",
    "print(qMatrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0,  64,  80,  64,   0,   0,   0,   0,   0],\n",
       "       [ 64,   0,  80,  64,  64,  80,   0,   0,   0],\n",
       "       [ 64,  64,   0,  64,  64,  80, 100,  80,  64],\n",
       "       [ 64,  64,  80,   0,   0,   0,   0,  80,  64],\n",
       "       [  0,  64,  80,   0,   0,  80,   0,   0,   0],\n",
       "       [  0,  64,  80,   0,  64,   0, 100,  80,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,  80,  64,   0,  80, 100,   0,  64],\n",
       "       [  0,   0,  80,  64,   0,   0,   0,  80,   0]])"
      ]
     },
     "execution_count": 364,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qMatrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  0.  64.  80.  64.   0.   0.   0.   0.   0.]\n",
      " [ 64.   0.  80.  64.  64.  80.   0.   0.   0.]\n",
      " [ 64.  64.   0.  64.  64.  80. 100.  80.  64.]\n",
      " [ 64.  64.  80.   0.   0.   0.   0.  80.  64.]\n",
      " [  0.  64.  80.   0.   0.  80.   0.   0.   0.]\n",
      " [  0.  64.  80.   0.  64.   0. 100.  80.   0.]\n",
      " [  0.   0.   0.   0.   0.   0.   0.   0.   0.]\n",
      " [  0.   0.  80.  64.   0.  80. 100.   0.  64.]\n",
      " [  0.   0.  80.  64.   0.   0.   0.  80.   0.]]\n"
     ]
    }
   ],
   "source": [
    "qMatrix=(qMatrix/np.max(qMatrix))*100\n",
    "print(qMatrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -1,   0,   0,   0,  -1,  -1,  -1,  -1,  -1],\n",
       "       [  0,  -1,   0,   0,   0,   0,  -1,  -1,  -1],\n",
       "       [  0,   0,  -1,   0,   0,   0, 100,   0,   0],\n",
       "       [  0,   0,   0,  -1,  -1,  -1,  -1,   0,   0],\n",
       "       [ -1,   0,   0,  -1,  -1,   0,  -1,  -1,  -1],\n",
       "       [ -1,   0,   0,  -1,   0,  -1, 100,   0,  -1],\n",
       "       [ -1,  -1,   0,  -1,  -1,   0, 100,   0,  -1],\n",
       "       [ -1,  -1,   0,   0,  -1,   0, 100,  -1,   0],\n",
       "       [ -1,  -1,   0,   0,  -1,  -1,  -1,   0,  -1]])"
      ]
     },
     "execution_count": 366,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rMatrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Camino eficiente\n",
    "-----------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [],
   "source": [
    "def findOptimalWay(initialState, Q, objetive):\n",
    "    #paso1 \n",
    "    actualState = initialState\n",
    "    camino = [initialState]\n",
    "    contador=0\n",
    "    totalValue=0\n",
    "    #paso2\n",
    "    while(actualState!=objetive):\n",
    "        qRow = Q[actualState]\n",
    "        maxValue = np.max(qRow)\n",
    "        maxValueIndex = np.where(qRow == maxValue)\n",
    "        nextState=maxValueIndex[0][0]\n",
    "        print(\"nextState\",nextState)\n",
    "        camino.append(nextState)\n",
    "        #paso3\n",
    "        totalValue += maxValue \n",
    "        actualState=nextState\n",
    "        print(actualState)\n",
    "        contador=contador+1\n",
    "    return camino, contador, totalValue\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nextState 2\n",
      "2\n",
      "nextState 6\n",
      "6\n"
     ]
    }
   ],
   "source": [
    "[camino,contador,totalValue]=findOptimalWay(8, qMatrix, objetivo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[8, 2, 6]"
      ]
     },
     "execution_count": 369,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "camino"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}