-
Notifications
You must be signed in to change notification settings - Fork 0
/
learningAgents.py
executable file
·258 lines (221 loc) · 9.57 KB
/
learningAgents.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
# learningAgents.py
# -----------------
# Licensing Information: You are free to use or extend these projects for
# educational purposes provided that (1) you do not distribute or publish
# solutions, (2) you retain this notice, and (3) you provide clear
# attribution to UC Berkeley, including a link to http://ai.berkeley.edu.
#
# Attribution Information: The Pacman AI projects were developed at UC Berkeley.
# The core projects and autograders were primarily created by John DeNero
# (denero@cs.berkeley.edu) and Dan Klein (klein@cs.berkeley.edu).
# Student side autograding was added by Brad Miller, Nick Hay, and
# Pieter Abbeel (pabbeel@cs.berkeley.edu).
from game import Directions, Agent, Actions
import random,util,time
class ValueEstimationAgent(Agent):
"""
Abstract agent which assigns values to (state,action)
Q-Values for an environment. As well as a value to a
state and a policy given respectively by,
V(s) = max_{a in actions} Q(s,a)
policy(s) = arg_max_{a in actions} Q(s,a)
Both ValueIterationAgent and QLearningAgent inherit
from this agent. While a ValueIterationAgent has
a model of the environment via a MarkovDecisionProcess
(see mdp.py) that is used to estimate Q-Values before
ever actually acting, the QLearningAgent estimates
Q-Values while acting in the environment.
"""
def __init__(self, alpha=1.0, epsilon=0.05, gamma=0.8, numTraining = 10):
"""
Sets options, which can be passed in via the Pacman command line using -a alpha=0.5,...
alpha - learning rate
epsilon - exploration rate
gamma - discount factor
numTraining - number of training episodes, i.e. no learning after these many episodes
"""
self.alpha = float(alpha)
self.epsilon = float(epsilon)
self.discount = float(gamma)
self.numTraining = int(numTraining)
####################################
# Override These Functions #
####################################
def getQValue(self, state, action):
"""
Should return Q(state,action)
"""
util.raiseNotDefined()
def getValue(self, state):
"""
What is the value of this state under the best action?
Concretely, this is given by
V(s) = max_{a in actions} Q(s,a)
"""
util.raiseNotDefined()
def getPolicy(self, state):
"""
What is the best action to take in the state. Note that because
we might want to explore, this might not coincide with getAction
Concretely, this is given by
policy(s) = arg_max_{a in actions} Q(s,a)
If many actions achieve the maximal Q-value,
it doesn't matter which is selected.
"""
util.raiseNotDefined()
def getAction(self, state):
"""
state: can call state.getLegalActions()
Choose an action and return it.
"""
util.raiseNotDefined()
class ReinforcementAgent(ValueEstimationAgent):
"""
Abstract Reinforcemnt Agent: A ValueEstimationAgent
which estimates Q-Values (as well as policies) from experience
rather than a model
What you need to know:
- The environment will call
observeTransition(state,action,nextState,deltaReward),
which will call update(state, action, nextState, deltaReward)
which you should override.
- Use self.getLegalActions(state) to know which actions
are available in a state
"""
####################################
# Override These Functions #
####################################
def update(self, state, action, nextState, reward):
"""
This class will call this function, which you write, after
observing a transition and reward
"""
util.raiseNotDefined()
####################################
# Read These Functions #
####################################
def getLegalActions(self,state):
"""
Get the actions available for a given
state. This is what you should use to
obtain legal actions for a state
"""
return self.actionFn(state)
def observeTransition(self, state,action,nextState,deltaReward):
"""
Called by environment to inform agent that a transition has
been observed. This will result in a call to self.update
on the same arguments
NOTE: Do *not* override or call this function
"""
self.episodeRewards += deltaReward
self.update(state,action,nextState,deltaReward)
def startEpisode(self):
"""
Called by environment when new episode is starting
"""
self.lastState = None
self.lastAction = None
self.episodeRewards = 0.0
def stopEpisode(self):
"""
Called by environment when episode is done
"""
if self.episodesSoFar < self.numTraining:
self.accumTrainRewards += self.episodeRewards
else:
self.accumTestRewards += self.episodeRewards
self.episodesSoFar += 1
if self.episodesSoFar >= self.numTraining:
# Take off the training wheels
self.epsilon = 0.0 # no exploration
self.alpha = 0.0 # no learning
def isInTraining(self):
return self.episodesSoFar < self.numTraining
def isInTesting(self):
return not self.isInTraining()
def __init__(self, actionFn = None, numTraining=100, epsilon=0.5, alpha=0.5, gamma=1):
"""
actionFn: Function which takes a state and returns the list of legal actions
alpha - learning rate
epsilon - exploration rate
gamma - discount factor
numTraining - number of training episodes, i.e. no learning after these many episodes
"""
if actionFn == None:
actionFn = lambda state: state.getLegalActions()
self.actionFn = actionFn
self.episodesSoFar = 0
self.accumTrainRewards = 0.0
self.accumTestRewards = 0.0
self.numTraining = int(numTraining)
self.epsilon = float(epsilon)
self.alpha = float(alpha)
self.discount = float(gamma)
################################
# Controls needed for Crawler #
################################
def setEpsilon(self, epsilon):
self.epsilon = epsilon
def setLearningRate(self, alpha):
self.alpha = alpha
def setDiscount(self, discount):
self.discount = discount
def doAction(self,state,action):
"""
Called by inherited class when
an action is taken in a state
"""
self.lastState = state
self.lastAction = action
###################
# Pacman Specific #
###################
def observationFunction(self, state):
"""
This is where we ended up after our last action.
The simulation should somehow ensure this is called
"""
if not self.lastState is None:
reward = state.getScore() - self.lastState.getScore()
self.observeTransition(self.lastState, self.lastAction, state, reward)
return state
def registerInitialState(self, state):
self.startEpisode()
if self.episodesSoFar == 0:
print('Beginning %d episodes of Training' % (self.numTraining))
def final(self, state):
"""
Called by Pacman game at the terminal state
"""
deltaReward = state.getScore() - self.lastState.getScore()
self.observeTransition(self.lastState, self.lastAction, state, deltaReward)
self.stopEpisode()
# Make sure we have this var
if not 'episodeStartTime' in self.__dict__:
self.episodeStartTime = time.time()
if not 'lastWindowAccumRewards' in self.__dict__:
self.lastWindowAccumRewards = 0.0
self.lastWindowAccumRewards += state.getScore()
NUM_EPS_UPDATE = 100
if self.episodesSoFar % NUM_EPS_UPDATE == 0:
print('Reinforcement Learning Status:')
windowAvg = self.lastWindowAccumRewards / float(NUM_EPS_UPDATE)
if self.episodesSoFar <= self.numTraining:
trainAvg = self.accumTrainRewards / float(self.episodesSoFar)
print('\tCompleted %d out of %d training episodes' % (
self.episodesSoFar,self.numTraining))
print('\tAverage Rewards over all training: %.2f' % (
trainAvg))
else:
testAvg = float(self.accumTestRewards) / (self.episodesSoFar - self.numTraining)
print('\tCompleted %d test episodes' % (self.episodesSoFar - self.numTraining))
print('\tAverage Rewards over testing: %.2f' % testAvg)
print('\tAverage Rewards for last %d episodes: %.2f' % (
NUM_EPS_UPDATE,windowAvg))
print('\tEpisode took %.2f seconds' % (time.time() - self.episodeStartTime))
self.lastWindowAccumRewards = 0.0
self.episodeStartTime = time.time()
if self.episodesSoFar == self.numTraining:
msg = 'Training Done (turning off epsilon and alpha)'
print('%s\n%s' % (msg,'-' * len(msg)))