Repository navigation
Reinforcement Learning: From Curiosity to Mastery
Ahsan Kabir edited this page Dec 23, 2025
·
1 revision
Table of Contents
- Part I: The Foundation - Thinking Like an Agent
- Part II: Classical Methods - Learning by Interaction
- Part III: Scaling Up - The Deep Learning Revolution
- Part IV: Advanced Strategies & The Real World
- Appendices
| Learning Type | Analogy | Real-World Example | Key Characteristic |
|---|---|---|---|
| Supervised Learning | Recipe follower with labeled ingredients | Spam filter | Needs correct answers |
| Unsupervised Learning | Ingredient explorer finding patterns | Customer segmentation | Discovers structure |
| Reinforcement Learning | Experimental chef tasting dishes | Game-playing AI | Learns from trial and error |
The Treasure Hunt Game Analogy:
import numpy as np
class TreasureHunt:
"""Simple RL environment demonstrating core concepts"""
def __init__(self):
# Agent: The treasure hunter (you!)
self.agent_position = "start_camp"
# Environment: The world map
self.map = {
"start_camp": {"north": "dark_forest", "east": "river"},
"dark_forest": {"south": "start_camp", "east": "mysterious_cave"},
"river": {"west": "start_camp", "north": "waterfall"},
"mysterious_cave": {"west": "dark_forest", "down": "treasure_room"},
"waterfall": {"south": "river", "behind": "hidden_grove"},
"hidden_grove": {"enter": "ancient_temple"},
"treasure_room": {}, # Terminal state
"ancient_temple": {} # Terminal state
}
# Rewards: Feedback signals
self.rewards = {
"start_camp": 0,
"dark_forest": -1, # Spooky, lose energy
"river": -1, # Crossing costs energy
"mysterious_cave": -2, # Dark and dangerous
"waterfall": 0,
"hidden_grove": 5, # Beautiful discovery
"treasure_room": 100, # JACKPOT!
"ancient_temple": 50 # Great discovery
}
# Actions available at each state
self.available_actions = {
"start_camp": ["north", "east"],
"dark_forest": ["south", "east"],
"river": ["west", "north"],
"mysterious_cave": ["west", "down"],
"waterfall": ["south", "behind"],
"hidden_grove": ["enter"],
"treasure_room": [],
"ancient_temple": []
}
def step(self, action):
"""Take an action, return new state and reward"""
current_state = self.agent_position
if action not in self.available_actions[current_state]:
print(f"Invalid action '{action}' from state '{current_state}'!")
return current_state, -10, False # Penalty for invalid action
# Determine next state with some randomness (real environments are stochastic!)
if np.random.random() < 0.1: # 10% chance of getting lost
possible_states = list(self.available_actions.keys())
next_state = np.random.choice(possible_states)
print("Oops! You got lost and ended up somewhere unexpected!")
else:
# Normal transition
next_state = self.map[current_state][action]
self.agent_position = next_state
reward = self.rewards[next_state]
# Check if terminal state
done = (next_state in ["treasure_room", "ancient_temple"])
return next_state, reward, done
def reset(self):
"""Reset the game"""
self.agent_position = "start_camp"
return self.agent_positionThe Multi-Armed Bandit Problem:
import numpy as np
import matplotlib.pyplot as plt
class MultiArmedBandit:
"""Simple exploration-exploitation problem"""
def __init__(self, n_arms=3):
# Each arm has a true reward probability (unknown to agent)
self.true_probs = np.random.random(n_arms)
print(f"True probabilities (secret!): {self.true_probs}")
# Agent's estimates
self.estimates = np.zeros(n_arms)
self.counts = np.zeros(n_arms)
def pull_arm(self, arm):
"""Pull an arm, return reward (0 or 1)"""
reward = 1 if np.random.random() < self.true_probs[arm] else 0
return reward
def update_estimate(self, arm, reward):
"""Update estimate using sample average"""
self.counts[arm] += 1
n = self.counts[arm]
self.estimates[arm] = ((n-1) * self.estimates[arm] + reward) / n
def epsilon_greedy(self, epsilon=0.1):
"""Epsilon-greedy action selection"""
if np.random.random() < epsilon:
# Explore: random arm
return np.random.randint(len(self.true_probs))
else:
# Exploit: best estimated arm
return np.argmax(self.estimates)
def run_bandit_experiment():
"""Run a bandit experiment"""
bandit = MultiArmedBandit(n_arms=5)
total_reward = 0
rewards_history = []
for t in range(1000):
# Use epsilon-greedy strategy
epsilon = max(0.01, 1.0 / (t/100 + 1)) # Decreasing epsilon
action = bandit.epsilon_greedy(epsilon)
# Get reward and update
reward = bandit.pull_arm(action)
bandit.update_estimate(action, reward)
total_reward += reward
rewards_history.append(reward)
# Plot results
plt.figure(figsize=(10, 5))
plt.subplot(1, 2, 1)
cumulative_rewards = np.cumsum(rewards_history)
plt.plot(cumulative_rewards)
plt.xlabel('Steps')
plt.ylabel('Cumulative Reward')
plt.title('Cumulative Rewards Over Time')
plt.grid(True, alpha=0.3)
plt.subplot(1, 2, 2)
plt.bar(range(len(bandit.estimates)), bandit.estimates)
plt.xlabel('Arm')
plt.ylabel('Estimated Value')
plt.title('Final Arm Estimates')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
return total_reward, bandit.estimates| Domain | Application | Key RL Component |
|---|---|---|
| Healthcare | Personalized treatment plans | Reward = Patient health improvement |
| Finance | Algorithmic trading | Action = Buy/Sell/Hold decisions |
| Robotics | Robot manipulation | State = Sensor readings |
| Games | AlphaGo, Dota 2 | Policy = Game strategy |
| Autonomous Vehicles | Self-driving cars | Transition = Road dynamics |
class GridWorld:
"""A simple grid world environment for beginners"""
def __init__(self, size=5):
self.size = size
self.state = (0, 0) # Start at top-left
self.goal = (size-1, size-1) # Goal at bottom-right
# Define obstacles (walls)
self.obstacles = [(1, 1), (2, 2), (3, 1)]
# Define rewards
self.rewards = {
self.goal: 10, # Goal reward
(2, 3): -5, # Trap
(1, 3): 2, # Small positive reward
}
# Actions: 0=up, 1=right, 2=down, 3=left
self.actions = [0, 1, 2, 3]
self.action_map = {
0: (-1, 0), # up
1: (0, 1), # right
2: (1, 0), # down
3: (0, -1) # left
}
def reset(self):
"""Reset environment to start state"""
self.state = (0, 0)
return self.state
def step(self, action):
"""Take action, return (next_state, reward, done)"""
current_row, current_col = self.state
# Get movement
row_change, col_change = self.action_map[action]
new_row = current_row + row_change
new_col = current_col + col_change
# Check boundaries
if new_row < 0 or new_row >= self.size or new_col < 0 or new_col >= self.size:
new_row, new_col = current_row, current_col # Stay in place
# Check obstacles
if (new_row, new_col) in self.obstacles:
new_row, new_col = current_row, current_col # Stay in place
self.state = (new_row, new_col)
# Get reward
reward = self.rewards.get((new_row, new_col), -0.1) # Small penalty for each step
# Check if done
done = (self.state == self.goal)
return self.state, reward, done
def render(self):
"""Print the current grid state"""
grid = [['.' for _ in range(self.size)] for _ in range(self.size)]
# Mark obstacles
for r, c in self.obstacles:
grid[r][c] = 'X'
# Mark goal
grid[self.goal[0]][self.goal[1]] = 'G'
# Mark agent
agent_r, agent_c = self.state
grid[agent_r][agent_c] = 'A'
# Print grid
print('+' + '---' * self.size + '+')
for row in grid:
print('|' + ' '.join(f' {cell} ' for cell in row) + '|')
print('+' + '---' * self.size + '+')
# Legend
print("\nLegend: A = Agent, G = Goal, X = Obstacle, . = Empty")Try running the GridWorld:
# Create and test the GridWorld
env = GridWorld(size=5)
state = env.reset()
print("Initial State:")
env.render()
# Try some actions
actions = [1, 1, 2, 2, 1, 2] # right, right, down, down, right, down
for i, action in enumerate(actions):
print(f"\nStep {i+1}: Taking action {action}")
state, reward, done = env.step(action)
env.render()
print(f"Reward: {reward}, Done: {done}")
if done:
print("Goal reached!")
breakFormal Definition:
An MDP is defined by the 5-tuple:
-
$S$ : Set of states -
$A$ : Set of actions -
$P$ : Transition probability$P(s'|s,a)$ -
$R$ : Reward function$R(s,a,s')$ -
$\gamma$ : Discount factor$(0 \leq \gamma \leq 1)$
class SimpleMDP:
"""Implementation of a simple MDP for the Coffee Shop Problem"""
def __init__(self):
# Define states
self.states = ['home', 'walking', 'shop', 'work']
# Define actions
self.actions = ['stay', 'go_to_shop', 'go_to_work', 'buy_coffee']
# Transition probabilities P(s'|s,a)
self.transitions = {
('home', 'go_to_shop'): {'walking': 1.0},
('walking', 'stay'): {'shop': 0.8, 'walking': 0.2}, # 20% chance of getting lost
('shop', 'buy_coffee'): {'work': 1.0},
('work', 'stay'): {'work': 1.0},
# Invalid actions lead back to same state
('home', 'stay'): {'home': 1.0},
('walking', 'go_to_work'): {'walking': 1.0}, # Can't go directly to work from walking
}
# Rewards R(s,a,s')
self.rewards = {
('home', 'go_to_shop', 'walking'): -1, # Effort to leave home
('walking', 'stay', 'shop'): +5, # Successfully reached shop
('walking', 'stay', 'walking'): -2, # Got lost
('shop', 'buy_coffee', 'work'): +10, # Got coffee and went to work
('work', 'stay', 'work'): +1, # Being productive
('home', 'stay', 'home'): 0, # Doing nothing
}
# Discount factor
self.gamma = 0.9
# Current state
self.current_state = 'home'
def get_possible_actions(self, state):
"""Get valid actions for a given state"""
valid_actions = []
for action in self.actions:
if (state, action) in self.transitions:
valid_actions.append(action)
return valid_actions
def step(self, action):
"""Take action, return (next_state, reward)"""
current_state = self.current_state
# Check if action is valid
if (current_state, action) not in self.transitions:
print(f"Invalid action '{action}' from state '{current_state}'")
return current_state, 0
# Get transition probabilities
next_state_probs = self.transitions[(current_state, action)]
# Sample next state
next_states = list(next_state_probs.keys())
probabilities = list(next_state_probs.values())
next_state = np.random.choice(next_states, p=probabilities)
# Get reward
reward = self.rewards.get((current_state, action, next_state), 0)
# Update state
self.current_state = next_state
return next_state, reward
def reset(self):
"""Reset MDP to initial state"""
self.current_state = 'home'
return self.current_state
def simulate_episode(self, policy):
"""Simulate one episode following a given policy"""
state = self.reset()
total_reward = 0
episode_history = []
for step in range(20): # Max 20 steps
# Get action from policy
if state in policy:
action = policy[state]
else:
# Default to 'stay' if no policy defined
action = 'stay'
# Take action
next_state, reward = self.step(action)
# Record step
episode_history.append({
'step': step,
'state': state,
'action': action,
'reward': reward,
'next_state': next_state
})
total_reward += (self.gamma ** step) * reward
state = next_state
# Check if terminal state (work)
if state == 'work':
break
return episode_history, total_rewardDefinition:
Example: Traffic Light MDP
class TrafficLightMDP:
"""MDP for traffic light control"""
def __init__(self):
# States: (light_color, queue_length)
# light_color: 0=red, 1=green, 2=yellow
# queue_length: 0-10 cars
self.states = [(color, length) for color in range(3) for length in range(11)]
# Actions: 0=keep_current, 1=switch_to_red, 2=switch_to_green
self.actions = [0, 1, 2]
# Transition probabilities
self.transitions = self._build_transition_model()
# Rewards
self.rewards = self._build_reward_model()
self.gamma = 0.9
def _build_transition_model(self):
"""Build transition probability model"""
transitions = {}
for state in self.states:
color, queue = state
for action in self.actions:
next_state_probs = {}
if action == 0: # Keep current
next_color = color
elif action == 1: # Switch to red
next_color = 0
else: # Switch to green
next_color = 1
# Queue dynamics
# Cars arrive randomly
arrival_rate = 0.3
# Cars depart if light is green
departure_rate = 0.8 if next_color == 1 else 0
# Simplified: only consider +/- 1 car changes
for delta in [-1, 0, 1]:
new_queue = max(0, min(10, queue + delta))
if delta == -1: # Car leaves
prob = departure_rate * 0.5
elif delta == 1: # Car arrives
prob = arrival_rate
else: # No change
prob = 1 - departure_rate * 0.5 - arrival_rate
next_state = (next_color, new_queue)
next_state_probs[next_state] = prob
transitions[(state, action)] = next_state_probs
return transitions
def _build_reward_model(self):
"""Build reward model"""
rewards = {}
for (state, action), next_states in self.transitions.items():
color, queue = state
for next_state, prob in next_states.items():
next_color, next_queue = next_state
# Reward: negative for long queues, negative for switching lights
reward = -queue # Penalty for current queue length
if action != 0: # Penalty for switching
reward -= 2
if next_color == 1 and next_queue > 0: # Bonus for serving cars
reward += 5
rewards[(state, action, next_state)] = reward
return rewards
def visualize_state_space(self):
"""Visualize the state space"""
import matplotlib.pyplot as plt
fig, axes = plt.subplots(1, 3, figsize=(15, 5))
colors = ['red', 'green', 'yellow']
for color_idx, color_name in enumerate(colors):
ax = axes[color_idx]
# Extract states for this color
states_for_color = [(q, 0) for q in range(11)]
# Simple visualization
ax.bar(range(11), [q for q in range(11)])
ax.set_xlabel('Queue Length')
ax.set_ylabel('State Value')
ax.set_title(f'Light Color: {color_name}')
ax.set_ylim(0, 10)
plt.tight_layout()
plt.show()Why Discount Future Rewards?
- Mathematical convenience: Guarantees finite returns
- Economic sense: Money now is worth more than money later
- Uncertainty: Distant future is less certain
- Biological basis: Animals prefer immediate rewards
Return Calculation:
def calculate_return(rewards, gamma=0.9):
"""
Calculate discounted return from a sequence of rewards
Return = R₁ + γR₂ + γ²R₃ + γ³R₄ + ...
"""
total = 0
for i, reward in enumerate(rewards):
total += (gamma ** i) * reward
return total
def calculate_returns_from_history(history, gamma=0.9):
"""Calculate returns from episode history"""
returns = []
G = 0
# Work backwards from the end
for step in reversed(history):
G = step['reward'] + gamma * G
returns.insert(0, G) # Insert at beginning
return returns
# Example usage
rewards = [1, 2, 3, 4, 5]
print(f"Undiscounted return: {sum(rewards)}")
print(f"Discounted return (γ=0.9): {calculate_return(rewards, gamma=0.9)}")
print(f"Discounted return (γ=0.5): {calculate_return(rewards, gamma=0.5)}")class Policy:
"""Base class for policies"""
def __init__(self, mdp):
self.mdp = mdp
def select_action(self, state):
"""Select action given state - to be implemented by subclasses"""
raise NotImplementedError
def update(self, state, action, reward, next_state):
"""Update policy based on experience"""
pass
class RandomPolicy(Policy):
"""Select actions randomly"""
def select_action(self, state):
valid_actions = self.mdp.get_possible_actions(state)
return np.random.choice(valid_actions)
class GreedyPolicy(Policy):
"""Select best action based on Q-values"""
def __init__(self, mdp, Q):
super().__init__(mdp)
self.Q = Q # Q-value table
def select_action(self, state):
valid_actions = self.mdp.get_possible_actions(state)
if not valid_actions:
return None
# Find action with maximum Q-value
q_values = [self.Q.get((state, action), 0) for action in valid_actions]
max_q = max(q_values)
# If multiple actions have same Q-value, choose randomly among them
best_actions = [action for action, q in zip(valid_actions, q_values) if q == max_q]
return np.random.choice(best_actions)
class EpsilonGreedyPolicy(Policy):
"""Epsilon-greedy policy for exploration"""
def __init__(self, mdp, Q, epsilon=0.1):
super().__init__(mdp)
self.Q = Q
self.epsilon = epsilon
def select_action(self, state):
valid_actions = self.mdp.get_possible_actions(state)
if not valid_actions:
return None
if np.random.random() < self.epsilon:
# Explore: random action
return np.random.choice(valid_actions)
else:
# Exploit: greedy action
q_values = [self.Q.get((state, action), 0) for action in valid_actions]
max_q = max(q_values)
best_actions = [action for action, q in zip(valid_actions, q_values) if q == max_q]
return np.random.choice(best_actions)
def decay_epsilon(self, decay_rate=0.995, min_epsilon=0.01):
"""Decay exploration rate over time"""
self.epsilon = max(min_epsilon, self.epsilon * decay_rate)class PizzaDeliveryMDP:
"""MDP for pizza delivery optimization"""
def __init__(self, n_locations=5):
self.n_locations = n_locations
# States: (current_location, has_pizza, time_remaining)
self.states = []
for loc in range(n_locations):
for has_pizza in [0, 1]:
for time_rem in range(6): # 0-5 time units remaining
self.states.append((loc, has_pizza, time_rem))
# Actions: 0=wait, 1=move_to_loc_0, ..., n_locations-1=move_to_loc_n-1
self.actions = list(range(n_locations + 1)) # +1 for wait
# Special locations
self.pizzeria = 0
self.delivery_locations = list(range(1, n_locations))
# Build transition and reward models
self.transitions = self._build_transitions()
self.rewards = self._build_rewards()
self.gamma = 0.9
def _build_transitions(self):
"""Build transition probabilities"""
transitions = {}
for state in self.states:
current_loc, has_pizza, time_rem = state
for action in self.actions:
next_state_probs = {}
if time_rem == 0:
# Terminal state - stay here forever
next_state_probs[state] = 1.0
else:
# Time decreases by 1
new_time = time_rem - 1
if action == 0: # Wait
new_loc = current_loc
new_has_pizza = has_pizza
next_state_probs[(new_loc, new_has_pizza, new_time)] = 1.0
else: # Move to location (action-1)
target_loc = action - 1
# 80% chance of successful move, 20% chance of getting stuck
if np.random.random() < 0.8:
new_loc = target_loc
else:
new_loc = current_loc
# Update pizza status
if new_loc == self.pizzeria:
new_has_pizza = 1 # Pick up pizza
elif new_loc in self.delivery_locations and has_pizza:
new_has_pizza = 0 # Deliver pizza
else:
new_has_pizza = has_pizza # Keep current status
next_state_probs[(new_loc, new_has_pizza, new_time)] = 1.0
transitions[(state, action)] = next_state_probs
return transitions
def _build_rewards(self):
"""Build reward function"""
rewards = {}
for (state, action), next_states in self.transitions.items():
current_loc, has_pizza, time_rem = state
for next_state, prob in next_states.items():
new_loc, new_has_pizza, new_time = next_state
# Base reward
reward = -1 # Cost for each time step
# Delivery reward
if has_pizza and new_has_pizza == 0 and current_loc != new_loc:
# Successfully delivered pizza
reward += 20
# Pickup reward (smaller than delivery)
if has_pizza == 0 and new_has_pizza == 1 and current_loc != new_loc:
# Successfully picked up pizza
reward += 5
# Timeout penalty
if new_time == 0 and new_has_pizza == 1:
# Failed to deliver before timeout
reward -= 50
rewards[(state, action, next_state)] = reward
return rewards
def simulate_delivery(self, policy, max_steps=20):
"""Simulate one delivery episode"""
# Start at pizzeria without pizza, with 5 time units
state = (self.pizzeria, 0, 5)
total_reward = 0
history = []
for step in range(max_steps):
current_loc, has_pizza, time_rem = state
# Get action from policy
action = policy.select_action(state)
if action is None:
break
# Take action
next_states = self.transitions[(state, action)]
next_state = np.random.choice(
list(next_states.keys()),
p=list(next_states.values())
)
reward = self.rewards[(state, action, next_state)]
# Record
history.append({
'step': step,
'state': state,
'action': action,
'reward': reward,
'next_state': next_state
})
total_reward += (self.gamma ** step) * reward
state = next_state
# Check termination
_, has_pizza, time_rem = state
if time_rem == 0:
break
return history, total_reward
# Create and test the MDP
pizza_mdp = PizzaDeliveryMDP(n_locations=5)
# Test random policy
random_policy = RandomPolicy(pizza_mdp)
history, total_reward = pizza_mdp.simulate_delivery(random_policy)
print(f"Random Policy Total Reward: {total_reward:.2f}")
print("\nDelivery History:")
for step in history[:5]: # Show first 5 steps
print(f"Step {step['step']}: At location {step['state'][0]}, "
f"Pizza: {'Yes' if step['state'][1] else 'No'}, "
f"Action: {step['action']}, Reward: {step['reward']}")State-Value Function
Action-Value Function
class ValueFunctions:
"""Implementation of value functions"""
@staticmethod
def calculate_state_value(mdp, state, policy, gamma=0.9, depth=0, max_depth=10):
"""
Calculate V^π(s) recursively
V^π(s) = Σ_a π(a|s) Σ_s' P(s'|s,a) [R(s,a,s') + γV^π(s')]
"""
if depth > max_depth:
return 0
state_value = 0
# Get valid actions
valid_actions = mdp.get_possible_actions(state)
for action in valid_actions:
# Action probability under policy
action_prob = policy.get_action_probability(state, action)
if action_prob == 0:
continue
# Sum over possible next states
next_state_probs = mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = mdp.rewards.get((state, action, next_state), 0)
# Recursive call for next state's value
next_state_value = ValueFunctions.calculate_state_value(
mdp, next_state, policy, gamma, depth + 1, max_depth
)
state_value += action_prob * trans_prob * (reward + gamma * next_state_value)
return state_value
@staticmethod
def calculate_action_value(mdp, state, action, policy, gamma=0.9, depth=0, max_depth=10):
"""
Calculate Q^π(s,a) recursively
Q^π(s,a) = Σ_s' P(s'|s,a) [R(s,a,s') + γV^π(s')]
"""
if depth > max_depth:
return 0
action_value = 0
next_state_probs = mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = mdp.rewards.get((state, action, next_state), 0)
# Calculate next state's value
next_state_value = ValueFunctions.calculate_state_value(
mdp, next_state, policy, gamma, depth + 1, max_depth
)
action_value += trans_prob * (reward + gamma * next_state_value)
return action_value
@staticmethod
def compute_value_table(mdp, policy, gamma=0.9, max_depth=10):
"""Compute V^π(s) for all states"""
V = {}
for state in mdp.states:
V[state] = ValueFunctions.calculate_state_value(
mdp, state, policy, gamma, 0, max_depth
)
return V
@staticmethod
def compute_q_table(mdp, policy, gamma=0.9, max_depth=10):
"""Compute Q^π(s,a) for all state-action pairs"""
Q = {}
for state in mdp.states:
valid_actions = mdp.get_possible_actions(state)
for action in valid_actions:
Q[(state, action)] = ValueFunctions.calculate_action_value(
mdp, state, action, policy, gamma, 0, max_depth
)
return QBellman Expectation Equation for
V^π(s) = Σ_a π(a|s) Σ_s' P(s'|s,a) [R(s,a,s') + γV^π(s')]
Bellman Expectation Equation for
Q^π(s,a) = Σ_s' P(s'|s,a) [R(s,a,s') + γ Σ_a' π(a'|s') Q^π(s',a')]
def bellman_expectation_iteration(mdp, policy, gamma=0.9, threshold=1e-4):
"""
Solve Bellman expectation equations through iteration
Returns: V^π for all states
"""
# Initialize value function
V = {state: 0 for state in mdp.states}
iteration = 0
while True:
delta = 0
new_V = {}
for state in mdp.states:
state_value = 0
# Get valid actions
valid_actions = mdp.get_possible_actions(state)
for action in valid_actions:
# Action probability under policy
action_prob = policy.get_action_probability(state, action)
if action_prob == 0:
continue
# Sum over possible next states
next_state_probs = mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = mdp.rewards.get((state, action, next_state), 0)
state_value += action_prob * trans_prob * (reward + gamma * V[next_state])
new_V[state] = state_value
delta = max(delta, abs(new_V[state] - V[state]))
# Update values
V = new_V
iteration += 1
# Check convergence
if delta < threshold:
print(f"Converged after {iteration} iterations with delta={delta:.6f}")
break
if iteration % 100 == 0:
print(f"Iteration {iteration}, delta={delta:.6f}")
return V
def bellman_optimality_iteration(mdp, gamma=0.9, threshold=1e-4):
"""
Solve Bellman optimality equations through iteration
Returns: V* and π* for all states
"""
# Initialize value function
V = {state: 0 for state in mdp.states}
iteration = 0
while True:
delta = 0
new_V = {}
for state in mdp.states:
# Find maximum over actions
max_value = float('-inf')
# Get valid actions
valid_actions = mdp.get_possible_actions(state)
for action in valid_actions:
action_value = 0
# Sum over possible next states
next_state_probs = mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = mdp.rewards.get((state, action, next_state), 0)
action_value += trans_prob * (reward + gamma * V[next_state])
max_value = max(max_value, action_value)
new_V[state] = max_value
delta = max(delta, abs(new_V[state] - V[state]))
# Update values
V = new_V
iteration += 1
# Check convergence
if delta < threshold:
print(f"Converged after {iteration} iterations with delta={delta:.6f}")
break
if iteration % 100 == 0:
print(f"Iteration {iteration}, delta={delta:.6f}")
# Extract optimal policy from optimal values
optimal_policy = {}
for state in mdp.states:
best_action = None
best_value = float('-inf')
valid_actions = mdp.get_possible_actions(state)
for action in valid_actions:
action_value = 0
# Calculate Q-value for this state-action pair
next_state_probs = mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = mdp.rewards.get((state, action, next_state), 0)
action_value += trans_prob * (reward + gamma * V[next_state])
if action_value > best_value:
best_value = action_value
best_action = action
if best_action is not None:
optimal_policy[state] = {best_action: 1.0}
else:
# No valid actions (terminal state)
optimal_policy[state] = {}
return V, optimal_policyOptimal Value Functions:
$V^*(s) = \max_\pi V^\pi(s)$ $Q^*(s,a) = \max_\pi Q^\pi(s,a)$
Optimal Policy: $\pi^(s) = \arg\max_a Q^(s,a)$
class OptimalPolicyFinder:
"""Methods for finding optimal policies"""
@staticmethod
def value_iteration(mdp, gamma=0.9, threshold=1e-4):
"""Value iteration algorithm"""
return bellman_optimality_iteration(mdp, gamma, threshold)
@staticmethod
def policy_iteration(mdp, gamma=0.9, threshold=1e-4):
"""Policy iteration algorithm"""
# Start with random policy
policy = {}
for state in mdp.states:
valid_actions = mdp.get_possible_actions(state)
if valid_actions:
# Uniform random policy
policy[state] = {action: 1.0/len(valid_actions) for action in valid_actions}
else:
policy[state] = {}
iteration = 0
while True:
iteration += 1
# Policy Evaluation
V = bellman_expectation_iteration(mdp, policy, gamma, threshold)
# Policy Improvement
policy_stable = True
new_policy = {}
for state in mdp.states:
valid_actions = mdp.get_possible_actions(state)
if not valid_actions:
new_policy[state] = {}
continue
# Find best action(s)
best_actions = []
best_value = float('-inf')
for action in valid_actions:
action_value = 0
# Calculate Q-value
next_state_probs = mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = mdp.rewards.get((state, action, next_state), 0)
action_value += trans_prob * (reward + gamma * V[next_state])
if action_value > best_value:
best_value = action_value
best_actions = [action]
elif abs(action_value - best_value) < 1e-6:
best_actions.append(action)
# Greedy policy: equal probability for all best actions
new_action_dist = {action: 1.0/len(best_actions) for action in best_actions}
new_policy[state] = new_action_dist
# Check if policy changed
if state in policy:
old_best = max(policy[state].items(), key=lambda x: x[1])[0]
new_best = max(new_action_dist.items(), key=lambda x: x[1])[0]
if old_best != new_best:
policy_stable = False
policy = new_policy
if policy_stable:
print(f"Policy converged after {iteration} iterations")
break
# Convert to deterministic policy
deterministic_policy = {}
for state, action_dist in policy.items():
if action_dist:
best_action = max(action_dist.items(), key=lambda x: x[1])[0]
deterministic_policy[state] = best_action
else:
deterministic_policy[state] = None
return V, deterministic_policy
@staticmethod
def compare_algorithms(mdp, gamma=0.9):
"""Compare value iteration and policy iteration"""
import time
print("="*60)
print("COMPARING OPTIMAL POLICY FINDING ALGORITHMS")
print("="*60)
# Value Iteration
print("\n1. VALUE ITERATION")
start_time = time.time()
V_vi, policy_vi = OptimalPolicyFinder.value_iteration(mdp, gamma)
vi_time = time.time() - start_time
# Policy Iteration
print("\n2. POLICY ITERATION")
start_time = time.time()
V_pi, policy_pi = OptimalPolicyFinder.policy_iteration(mdp, gamma)
pi_time = time.time() - start_time
# Compare results
print("\n" + "="*60)
print("COMPARISON RESULTS")
print("="*60)
print(f"Value Iteration Time: {vi_time:.4f} seconds")
print(f"Policy Iteration Time: {pi_time:.4f} seconds")
# Check if policies are equivalent
policy_match = True
for state in mdp.states:
if state in policy_vi and state in policy_pi:
if policy_vi[state] != policy_pi[state]:
policy_match = False
print(f"Policy mismatch at state {state}: VI={policy_vi[state]}, PI={policy_pi[state]}")
if policy_match:
print("\n✓ Both algorithms found the same optimal policy!")
else:
print("\n✗ Algorithms found different policies")
# Compare value functions
value_diff = 0
for state in mdp.states:
if state in V_vi and state in V_pi:
diff = abs(V_vi[state] - V_pi[state])
value_diff = max(value_diff, diff)
print(f"Maximum value difference: {value_diff:.6f}")
return V_vi, policy_vi, V_pi, policy_piThe 3-State Maze Example:
def solve_3_state_maze_manually():
"""
Solve a simple 3-state maze manually
States: A -- B -- C
Start: A, Goal: C (reward +10)
Each move: cost -1
γ = 0.9
"""
print("="*60)
print("SOLVING 3-STATE MAZE MANUALLY")
print("="*60)
print("\nMaze Layout:")
print("A -- B -- C")
print("Start: A, Goal: C")
print("Move cost: -1, Goal reward: +10, γ = 0.9")
print("\n" + "-"*60)
print("STEP 1: Define terminal state values")
print("-"*60)
# Terminal state: C (goal reached)
V_C = 0 # No future rewards from terminal state
print(f"V(C) = 0 (terminal state)")
print("\n" + "-"*60)
print("STEP 2: Work backwards from the goal")
print("-"*60)
# State B: Can only go to C
print("\nState B:")
print("Only action: go to C")
print("Reward: -1 (move cost) + 10 (goal reward) = 9")
print("V(B) = R + γV(C)")
print(f" = 9 + 0.9 * 0")
V_B = 9
print(f" = {V_B}")
print("\n" + "-"*60)
print("STEP 3: Continue working backwards")
print("-"*60)
# State A: Can go to B
print("\nState A:")
print("Only action: go to B")
print("Reward: -1 (move cost)")
print("V(A) = R + γV(B)")
print(f" = -1 + 0.9 * {V_B}")
V_A = -1 + 0.9 * V_B
print(f" = {V_A:.1f}")
print("\n" + "-"*60)
print("FINAL VALUE FUNCTION")
print("-"*60)
print(f"V(A) = {V_A:.1f}")
print(f"V(B) = {V_B}")
print(f"V(C) = {V_C}")
print("\n" + "-"*60)
print("OPTIMAL POLICY")
print("-"*60)
print("From A: go to B (only choice)")
print("From B: go to C (only choice)")
print("From C: terminal state")
return {"A": V_A, "B": V_B, "C": V_C}
# Run the manual solution
maze_values = solve_3_state_maze_manually()def visualize_value_function(mdp, V, title="Value Function"):
"""Visualize value function for grid-like MDPs"""
import matplotlib.pyplot as plt
import numpy as np
# Check if states are grid coordinates
sample_state = next(iter(V.keys()))
if isinstance(sample_state, tuple) and len(sample_state) == 2:
# Extract grid dimensions
rows = max(s[0] for s in V.keys()) + 1
cols = max(s[1] for s in V.keys()) + 1
# Create grid
grid = np.zeros((rows, cols))
for (r, c), value in V.items():
grid[r, c] = value
# Plot
plt.figure(figsize=(10, 8))
# Heatmap
plt.subplot(1, 2, 1)
im = plt.imshow(grid, cmap='RdYlGn', aspect='auto')
plt.colorbar(im, label='State Value')
plt.title(f'{title} - Heatmap')
plt.xlabel('Column')
plt.ylabel('Row')
# Add value annotations
for i in range(rows):
for j in range(cols):
plt.text(j, i, f'{grid[i, j]:.1f}',
ha='center', va='center',
color='black' if abs(grid[i, j]) < 5 else 'white')
# 3D surface plot
plt.subplot(1, 2, 2, projection='3d')
X, Y = np.meshgrid(range(cols), range(rows))
surf = plt.gca().plot_surface(X, Y, grid, cmap='RdYlGn',
linewidth=0, antialiased=False)
plt.colorbar(surf, label='State Value', shrink=0.5)
plt.title(f'{title} - 3D Surface')
plt.xlabel('Column')
plt.ylabel('Row')
plt.gca().set_zlabel('Value')
plt.tight_layout()
plt.show()
else:
# Bar chart for non-grid states
plt.figure(figsize=(12, 6))
states = list(V.keys())
values = [V[s] for s in states]
# Convert states to strings for labeling
state_labels = [str(s) for s in states]
plt.bar(range(len(states)), values, color='steelblue', alpha=0.7)
plt.xticks(range(len(states)), state_labels, rotation=45, ha='right')
plt.ylabel('State Value')
plt.title(title)
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
def plot_value_iteration_convergence(mdp, gamma=0.9):
"""Plot convergence of value iteration"""
import matplotlib.pyplot as plt
V = {state: 0 for state in mdp.states}
deltas = []
values_history = []
iteration = 0
while True:
delta = 0
new_V = {}
for state in mdp.states:
max_value = float('-inf')
valid_actions = mdp.get_possible_actions(state)
for action in valid_actions:
action_value = 0
next_state_probs = mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = mdp.rewards.get((state, action, next_state), 0)
action_value += trans_prob * (reward + gamma * V[next_state])
max_value = max(max_value, action_value)
new_V[state] = max_value
delta = max(delta, abs(new_V[state] - V[state]))
V = new_V
deltas.append(delta)
values_history.append(V.copy())
iteration += 1
if delta < 1e-6 or iteration > 100:
break
# Plot convergence
plt.figure(figsize=(15, 5))
# Delta plot
plt.subplot(1, 3, 1)
plt.plot(deltas, 'b-', linewidth=2)
plt.yscale('log')
plt.xlabel('Iteration')
plt.ylabel('Max Delta (log scale)')
plt.title('Value Iteration Convergence')
plt.grid(True, alpha=0.3)
# Value evolution for some states
plt.subplot(1, 3, 2)
sample_states = list(mdp.states)[:5] # First 5 states
for i, state in enumerate(sample_states):
state_values = [values[state] for values in values_history]
plt.plot(state_values, label=f'State {state}', linewidth=2)
plt.xlabel('Iteration')
plt.ylabel('State Value')
plt.title('Value Evolution')
plt.legend()
plt.grid(True, alpha=0.3)
# Final value distribution
plt.subplot(1, 3, 3)
final_values = list(V.values())
plt.hist(final_values, bins=20, color='steelblue', alpha=0.7, edgecolor='black')
plt.xlabel('Final State Value')
plt.ylabel('Frequency')
plt.title('Distribution of Final Values')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print(f"Converged in {iteration} iterations")
print(f"Final max delta: {deltas[-1]:.6f}")
print(f"Average state value: {np.mean(final_values):.4f}")
return V, deltasclass TaxiWorld:
"""Taxi environment for value iteration project"""
def __init__(self, grid_size=4):
self.grid_size = grid_size
# States: (taxi_row, taxi_col, passenger_location, destination)
# passenger_location: 0=in taxi, 1=location1, 2=location2, 3=location3, 4=location4
# destination: 1-4
self.states = []
for r in range(grid_size):
for c in range(grid_size):
for passenger_loc in range(5): # 0-4
for destination in range(1, 5):
self.states.append((r, c, passenger_loc, destination))
# Actions: 0=up, 1=right, 2=down, 3=left, 4=pickup, 5=dropoff
self.actions = list(range(6))
# Special locations
self.passenger_locations = {
1: (0, 0),
2: (0, grid_size-1),
3: (grid_size-1, 0),
4: (grid_size-1, grid_size-1)
}
self.destinations = self.passenger_locations
# Build transition model
self.transitions = self._build_transition_model()
# Build reward model
self.rewards = self._build_reward_model()
self.gamma = 0.9
def _build_transition_model(self):
"""Build transition probabilities"""
transitions = {}
for state in self.states:
taxi_r, taxi_c, passenger_loc, destination = state
for action in self.actions:
next_state_probs = {}
if action < 4: # Movement
# Determine new taxi position
if action == 0: # up
new_r, new_c = max(0, taxi_r - 1), taxi_c
elif action == 1: # right
new_r, new_c = taxi_r, min(self.grid_size-1, taxi_c + 1)
elif action == 2: # down
new_r, new_c = min(self.grid_size-1, taxi_r + 1), taxi_c
else: # left
new_r, new_c = taxi_r, max(0, taxi_c - 1)
# Passenger location unchanged
new_passenger_loc = passenger_loc
elif action == 4: # pickup
new_r, new_c = taxi_r, taxi_c
# Check if taxi is at passenger location
if passenger_loc > 0: # Passenger at a location
passenger_loc_coords = self.passenger_locations[passenger_loc]
if (taxi_r, taxi_c) == passenger_loc_coords:
# Successful pickup
new_passenger_loc = 0 # Passenger in taxi
else:
# Failed pickup
new_passenger_loc = passenger_loc
else:
# Passenger already in taxi
new_passenger_loc = passenger_loc
else: # dropoff
new_r, new_c = taxi_r, taxi_c
# Check if passenger is in taxi and at destination
if passenger_loc == 0: # Passenger in taxi
destination_coords = self.destinations[destination]
if (taxi_r, taxi_c) == destination_coords:
# Successful dropoff
new_passenger_loc = destination # Passenger delivered
else:
# Failed dropoff
new_passenger_loc = passenger_loc
else:
# Passenger not in taxi
new_passenger_loc = passenger_loc
# Create next state
next_state = (new_r, new_c, new_passenger_loc, destination)
next_state_probs[next_state] = 1.0
transitions[(state, action)] = next_state_probs
return transitions
def _build_reward_model(self):
"""Build reward function"""
rewards = {}
for (state, action), next_states in self.transitions.items():
taxi_r, taxi_c, passenger_loc, destination = state
for next_state, prob in next_states.items():
new_r, new_c, new_passenger_loc, new_destination = next_state
# Base reward
reward = -1 # Cost for each step
# Pickup reward/penalty
if action == 4: # pickup
if passenger_loc > 0 and (taxi_r, taxi_c) == self.passenger_locations[passenger_loc]:
if new_passenger_loc == 0:
reward += 5 # Successful pickup
else:
reward -= 10 # Failed pickup
# Dropoff reward/penalty
elif action == 5: # dropoff
if passenger_loc == 0 and (taxi_r, taxi_c) == self.destinations[destination]:
if new_passenger_loc == destination:
reward += 20 # Successful dropoff
else:
reward -= 10 # Failed dropoff
rewards[(state, action, next_state)] = reward
return rewards
def get_possible_actions(self, state):
"""Get valid actions for a state"""
return self.actions # All actions valid in all states
def visualize_policy(self, policy):
"""Visualize policy for different passenger/destination configurations"""
fig, axes = plt.subplots(4, 4, figsize=(20, 16))
destination_names = {1: 'R', 2: 'G', 3: 'Y', 4: 'B'}
action_symbols = {
0: '↑', # up
1: '→', # right
2: '↓', # down
3: '←', # left
4: 'P', # pickup
5: 'D' # dropoff
}
for dest_idx, destination in enumerate(range(1, 5)):
for pass_idx, passenger_loc in enumerate(range(5)):
ax = axes[dest_idx, pass_idx]
# Create grid for this configuration
grid = np.full((self.grid_size, self.grid_size), '', dtype=object)
# Fill grid with actions
for r in range(self.grid_size):
for c in range(self.grid_size):
state = (r, c, passenger_loc, destination)
if state in policy:
action = policy[state]
if action is not None:
grid[r, c] = action_symbols.get(action, '?')
# Plot grid
im = ax.imshow(np.ones((self.grid_size, self.grid_size)),
cmap='Blues', vmin=0, vmax=1)
# Add action symbols
for r in range(self.grid_size):
for c in range(self.grid_size):
if grid[r, c]:
ax.text(c, r, grid[r, c],
ha='center', va='center',
fontsize=14, fontweight='bold')
# Mark special locations
for loc_num, (lr, lc) in self.passenger_locations.items():
color = ['red', 'green', 'yellow', 'blue'][loc_num-1]
ax.add_patch(plt.Rectangle((lc-0.5, lr-0.5), 1, 1,
fill=False, edgecolor=color,
linewidth=3))
# Title
if passenger_loc == 0:
pass_text = "In taxi"
else:
pass_text = f"At {destination_names[passenger_loc]}"
ax.set_title(f"Dest: {destination_names[destination]}, Pass: {pass_text}")
ax.set_xticks([])
ax.set_yticks([])
plt.suptitle("Taxi Policy Visualization", fontsize=16, fontweight='bold')
plt.tight_layout()
plt.show()
def taxi_value_iteration_project():
"""Complete project: Solve taxi problem with value iteration"""
print("="*70)
print("TAXI VALUE ITERATION PROJECT")
print("="*70)
# Create environment
print("\n1. Creating Taxi Environment...")
taxi_env = TaxiWorld(grid_size=4)
print(f" Number of states: {len(taxi_env.states):,}")
print(f" Number of actions: {len(taxi_env.actions)}")
# Run value iteration
print("\n2. Running Value Iteration...")
V, policy = OptimalPolicyFinder.value_iteration(taxi_env, gamma=0.9)
print(f"\n3. Results:")
print(f" Optimal value for sample state (0,0,1,4): {V.get((0,0,1,4), 'N/A'):.2f}")
print(f" Optimal action for sample state: {policy.get((0,0,1,4), 'N/A')}")
# Visualize
print("\n4. Visualizing Results...")
taxi_env.visualize_policy(policy)
# Test policy
print("\n5. Testing Policy...")
test_scenarios = [
((1, 1, 1, 4), "Taxi at (1,1), passenger at R, destination B"),
((3, 2, 0, 2), "Taxi at (3,2), passenger in taxi, destination G"),
((0, 3, 3, 1), "Taxi at (0,3), passenger at Y, destination R")
]
for state, description in test_scenarios:
if state in policy:
action = policy[state]
action_name = ['up', 'right', 'down', 'left', 'pickup', 'dropoff'][action]
print(f" {description}: Action = {action_name}")
else:
print(f" {description}: State not in policy")
return taxi_env, V, policy
# Run the project
if __name__ == "__main__":
taxi_env, V, policy = taxi_value_iteration_project()When you know P(s'|s,a) and R(s,a,s'), you can plan before acting!
class DynamicProgramming:
"""Collection of Dynamic Programming algorithms"""
def __init__(self, mdp):
self.mdp = mdp
def iterative_policy_evaluation(self, policy, gamma=0.9, theta=1e-4):
"""
Evaluate a policy through iteration
V_{k+1}(s) = Σ_a π(a|s) Σ_s' P(s'|s,a) [R(s,a,s') + γV_k(s')]
"""
# Initialize value function
V = {state: 0 for state in self.mdp.states}
iteration = 0
while True:
delta = 0
new_V = {}
for state in self.mdp.states:
v = V[state]
# Calculate new value using Bellman expectation equation
new_v = 0
valid_actions = self.mdp.get_possible_actions(state)
for action in valid_actions:
# Get action probability from policy
if state in policy and action in policy[state]:
action_prob = policy[state][action]
else:
action_prob = 0
if action_prob == 0:
continue
# Sum over next states
next_state_probs = self.mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = self.mdp.rewards.get((state, action, next_state), 0)
new_v += action_prob * trans_prob * (reward + gamma * V[next_state])
new_V[state] = new_v
delta = max(delta, abs(v - new_v))
# Update value function
V = new_V
iteration += 1
# Check convergence
if delta < theta:
print(f"Policy evaluation converged in {iteration} iterations (delta={delta:.6f})")
break
if iteration % 100 == 0:
print(f"Iteration {iteration}, delta={delta:.6f}")
return V
def policy_improvement(self, V, gamma=0.9):
"""
Improve policy by acting greedily with respect to V
π'(s) = argmax_a Σ_s' P(s'|s,a) [R(s,a,s') + γV(s')]
"""
new_policy = {}
for state in self.mdp.states:
best_action = None
best_value = float('-inf')
valid_actions = self.mdp.get_possible_actions(state)
for action in valid_actions:
# Calculate Q-value for this state-action pair
action_value = 0
next_state_probs = self.mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = self.mdp.rewards.get((state, action, next_state), 0)
action_value += trans_prob * (reward + gamma * V[next_state])
if action_value > best_value:
best_value = action_value
best_action = action
if best_action is not None:
new_policy[state] = {best_action: 1.0} # Deterministic policy
else:
new_policy[state] = {} # No valid actions
return new_policy
def policy_iteration(self, initial_policy=None, gamma=0.9, theta=1e-4):
"""
Full policy iteration algorithm
π₀ → Evaluate → Vπ₀ → Improve → π₁ → Evaluate → Vπ₁ → ...
"""
# Start with initial policy (random if not provided)
if initial_policy is None:
policy = self._create_random_policy()
else:
policy = initial_policy
iteration = 0
policy_history = [policy.copy()]
value_history = []
while True:
iteration += 1
print(f"\nPolicy Iteration {iteration}")
print("-" * 40)
# Policy Evaluation
print("Step 1: Policy Evaluation...")
V = self.iterative_policy_evaluation(policy, gamma, theta)
value_history.append(V.copy())
# Policy Improvement
print("Step 2: Policy Improvement...")
new_policy = self.policy_improvement(V, gamma)
# Check if policy has changed
policy_changed = False
for state in self.mdp.states:
if state in policy and state in new_policy:
old_best = max(policy[state].items(), key=lambda x: x[1])[0] if policy[state] else None
new_best = max(new_policy[state].items(), key=lambda x: x[1])[0] if new_policy[state] else None
if old_best != new_best:
policy_changed = True
break
policy = new_policy
policy_history.append(policy.copy())
# Check convergence
if not policy_changed:
print(f"\n✓ Policy converged after {iteration} iterations!")
break
return policy, V, policy_history, value_history
def value_iteration(self, gamma=0.9, theta=1e-4):
"""
Value iteration algorithm
V_{k+1}(s) = max_a Σ_s' P(s'|s,a) [R(s,a,s') + γV_k(s')]
"""
# Initialize value function
V = {state: 0 for state in self.mdp.states}
iteration = 0
value_history = [V.copy()]
deltas = []
while True:
delta = 0
new_V = {}
for state in self.mdp.states:
v = V[state]
# Find maximum over actions
max_value = float('-inf')
valid_actions = self.mdp.get_possible_actions(state)
for action in valid_actions:
action_value = 0
next_state_probs = self.mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = self.mdp.rewards.get((state, action, next_state), 0)
action_value += trans_prob * (reward + gamma * V[next_state])
max_value = max(max_value, action_value)
new_V[state] = max_value
delta = max(delta, abs(v - max_value))
# Update value function
V = new_V
value_history.append(V.copy())
deltas.append(delta)
iteration += 1
# Check convergence
if delta < theta:
print(f"Value iteration converged in {iteration} iterations (delta={delta:.6f})")
break
if iteration % 100 == 0:
print(f"Iteration {iteration}, delta={delta:.6f}")
# Extract optimal policy
optimal_policy = self.policy_improvement(V, gamma)
return optimal_policy, V, value_history, deltas
def _create_random_policy(self):
"""Create a random policy for initialization"""
policy = {}
for state in self.mdp.states:
valid_actions = self.mdp.get_possible_actions(state)
if valid_actions:
# Uniform random policy
action_prob = 1.0 / len(valid_actions)
policy[state] = {action: action_prob for action in valid_actions}
else:
policy[state] = {} # No valid actions
return policy
def compare_dp_algorithms(self, gamma=0.9):
"""Compare policy iteration and value iteration"""
import time
print("="*70)
print("COMPARING DYNAMIC PROGRAMMING ALGORITHMS")
print("="*70)
# Policy Iteration
print("\n1. POLICY ITERATION")
start_time = time.time()
pi_policy, pi_V, pi_history, pi_values = self.policy_iteration(gamma=gamma)
pi_time = time.time() - start_time
# Value Iteration
print("\n2. VALUE ITERATION")
start_time = time.time()
vi_policy, vi_V, vi_values, vi_deltas = self.value_iteration(gamma=gamma)
vi_time = time.time() - start_time
# Results
print("\n" + "="*70)
print("RESULTS")
print("="*70)
print(f"\nPolicy Iteration:")
print(f" Time: {pi_time:.3f} seconds")
print(f" Iterations: {len(pi_history) - 1}")
print(f"\nValue Iteration:")
print(f" Time: {vi_time:.3f} seconds")
print(f" Iterations: {len(vi_values) - 1}")
# Compare optimal values
print(f"\nValue Comparison:")
sample_states = list(self.mdp.states)[:5] # First 5 states
for state in sample_states:
pi_value = pi_V.get(state, 0)
vi_value = vi_V.get(state, 0)
diff = abs(pi_value - vi_value)
print(f" State {state}: PI={pi_value:.4f}, VI={vi_value:.4f}, Diff={diff:.6f}")
# Compare policies
print(f"\nPolicy Comparison:")
policy_match = True
for state in self.mdp.states:
if state in pi_policy and state in vi_policy:
pi_best = max(pi_policy[state].items(), key=lambda x: x[1])[0] if pi_policy[state] else None
vi_best = max(vi_policy[state].items(), key=lambda x: x[1])[0] if vi_policy[state] else None
if pi_best != vi_best:
policy_match = False
print(f" State {state}: PI={pi_best}, VI={vi_best}")
if policy_match:
print(" ✓ Policies match perfectly!")
else:
print(" ✗ Policies differ")
return pi_policy, pi_V, vi_policy, vi_Vdef demonstrate_policy_evaluation():
"""Step-by-step demonstration of policy evaluation"""
print("="*70)
print("POLICY EVALUATION DEMONSTRATION")
print("="*70)
# Create a simple MDP
class SimpleDemoMDP:
def __init__(self):
self.states = ['s1', 's2', 's3']
self.actions = ['a1', 'a2']
# Transition probabilities
self.transitions = {
('s1', 'a1'): {'s2': 0.7, 's1': 0.3},
('s1', 'a2'): {'s3': 1.0},
('s2', 'a1'): {'s3': 1.0},
('s2', 'a2'): {'s1': 0.5, 's2': 0.5},
('s3', 'a1'): {'s3': 1.0},
('s3', 'a2'): {'s1': 1.0}
}
# Rewards
self.rewards = {
('s1', 'a1', 's2'): 5,
('s1', 'a1', 's1'): -1,
('s1', 'a2', 's3'): 10,
('s2', 'a1', 's3'): -2,
('s2', 'a2', 's1'): 1,
('s2', 'a2', 's2'): -1,
('s3', 'a1', 's3'): 0,
('s3', 'a2', 's1'): 0
}
def get_possible_actions(self, state):
return [a for a in self.actions if (state, a) in self.transitions]
mdp = SimpleDemoMDP()
dp = DynamicProgramming(mdp)
# Define a policy to evaluate
policy = {
's1': {'a1': 0.6, 'a2': 0.4},
's2': {'a1': 0.3, 'a2': 0.7},
's3': {'a1': 0.5, 'a2': 0.5}
}
print("\nInitial Policy:")
for state, actions in policy.items():
print(f" π({state}) = {actions}")
print("\n" + "-"*70)
print("STEP 1: Initialize Value Function")
print("-"*70)
V = {state: 0 for state in mdp.states}
print(f"V₀ = {V}")
print("\n" + "-"*70)
print("STEP 2: First Iteration (k=1)")
print("-"*70)
gamma = 0.9
V1 = {}
for state in mdp.states:
print(f"\nCalculating V₁({state}):")
state_value = 0
for action, action_prob in policy[state].items():
print(f" Action {action} (probability {action_prob}):")
next_state_probs = mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = mdp.rewards.get((state, action, next_state), 0)
term = action_prob * trans_prob * (reward + gamma * V[next_state])
state_value += term
print(f" P({next_state}|{state},{action}) = {trans_prob}")
print(f" R({state},{action},{next_state}) = {reward}")
print(f" Contribution: {action_prob} × {trans_prob} × ({reward} + 0.9×{V[next_state]}) = {term:.3f}")
V1[state] = state_value
print(f" V₁({state}) = {state_value:.3f}")
print(f"\nV₁ = { {s: f'{v:.3f}' for s, v in V1.items()} }")
print("\n" + "-"*70)
print("STEP 3: Second Iteration (k=2)")
print("-"*70)
V2 = {}
for state in mdp.states:
state_value = 0
for action, action_prob in policy[state].items():
next_state_probs = mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = mdp.rewards.get((state, action, next_state), 0)
state_value += action_prob * trans_prob * (reward + gamma * V1[next_state])
V2[state] = state_value
print(f"V₂ = { {s: f'{v:.3f}' for s, v in V2.items()} }")
print("\n" + "-"*70)
print("STEP 4: Continue Until Convergence")
print("-"*70)
# Use the DP algorithm
final_V = dp.iterative_policy_evaluation(policy, gamma=0.9)
print(f"Final V^π = { {s: f'{v:.3f}' for s, v in final_V.items()} }")
print("\n" + "-"*70)
print("INTERPRETATION")
print("-"*70)
print("\nThe value function tells us:")
for state, value in final_V.items():
print(f" Starting from {state} and following policy π,")
print(f" the agent can expect {value:.2f} total discounted reward")
return mdp, policy, final_V
# Run the demonstration
mdp, policy, V = demonstrate_policy_evaluation()def demonstrate_policy_improvement():
"""Demonstrate the policy improvement theorem"""
print("="*70)
print("POLICY IMPROVEMENT THEOREM DEMONSTRATION")
print("="*70)
# Create a simple MDP
class ImprovementMDP:
def __init__(self):
self.states = ['A', 'B', 'C']
self.actions = ['left', 'right']
self.transitions = {
('A', 'left'): {'B': 1.0},
('A', 'right'): {'C': 1.0},
('B', 'left'): {'A': 1.0},
('B', 'right'): {'C': 1.0},
('C', 'left'): {'B': 1.0},
('C', 'right'): {'A': 1.0}
}
self.rewards = {
('A', 'left', 'B'): 1,
('A', 'right', 'C'): 2,
('B', 'left', 'A'): 3,
('B', 'right', 'C'): 4,
('C', 'left', 'B'): 5,
('C', 'right', 'A'): 6
}
def get_possible_actions(self, state):
return [a for a in self.actions if (state, a) in self.transitions]
mdp = ImprovementMDP()
dp = DynamicProgramming(mdp)
# Initial policy: always go left
π = {
'A': {'left': 1.0},
'B': {'left': 1.0},
'C': {'left': 1.0}
}
print("\nInitial Policy π (always go left):")
for state, actions in π.items():
print(f" π({state}) = {actions}")
print("\n" + "-"*70)
print("STEP 1: Evaluate Initial Policy")
print("-"*70)
gamma = 0.9
V_π = dp.iterative_policy_evaluation(π, gamma)
print(f"\nValue function V^π:")
for state, value in V_π.items():
print(f" V^π({state}) = {value:.3f}")
print("\n" + "-"*70)
print("STEP 2: Compute Q^π(s,a) for all state-action pairs")
print("-"*70)
Q_π = {}
for state in mdp.states:
for action in mdp.get_possible_actions(state):
q_value = 0
next_state_probs = mdp.transitions.get((state, action), {})
for next_state, trans_prob in next_state_probs.items():
reward = mdp.rewards.get((state, action, next_state), 0)
q_value += trans_prob * (reward + gamma * V_π[next_state])
Q_π[(state, action)] = q_value
print(f" Q^π({state}, {action}) = {q_value:.3f}")
print("\n" + "-"*70)
print("STEP 3: Create Greedy Policy π'")
print("-"*70)
π_prime = {}
for state in mdp.states:
valid_actions = mdp.get_possible_actions(state)
if valid_actions:
# Find action with maximum Q-value
q_values = [Q_π[(state, a)] for a in valid_actions]
max_q = max(q_values)
best_actions = [a for a, q in zip(valid_actions, q_values) if q == max_q]
# Deterministic greedy policy
π_prime[state] = {best_actions[0]: 1.0}
print(f" At state {state}:")
print(f" Q-values: { {a: Q_π[(state, a)]:.3f} for a in valid_actions }")
print(f" Best action(s): {best_actions}")
print(f" π'({state}) = {best_actions[0]}")
else:
π_prime[state] = {}
print("\n" + "-"*70)
print("STEP 4: Verify Policy Improvement Theorem")
print("-"*70)
print("\nThe Policy Improvement Theorem states:")
print("If Q^π(s, π'(s)) ≥ V^π(s) for all s, then V^π'(s) ≥ V^π(s) for all s")
print("\nChecking condition:")
for state in mdp.states:
if state in π_prime and π_prime[state]:
greedy_action = list(π_prime[state].keys())[0]
Q_greedy = Q_π[(state, greedy_action)]
V_current = V_π[state]
condition_met = Q_greedy >= V_current
symbol = "✓" if condition_met else "✗"
print(f" {symbol} State {state}: Q^π({state}, {greedy_action}) = {Q_greedy:.3f} ≥ V^π({state}) = {V_current:.3f}")
print("\n" + "-"*70)
print("STEP 5: Evaluate Improved Policy")
print("-"*70)
V_π_prime = dp.iterative_policy_evaluation(π_prime, gamma)
print(f"\nValue function V^π':")
for state, value in V_π_prime.items():
print(f" V^π'({state}) = {value:.3f}")
print(f"\nValue function V^π:")
for state, value in V_π.items():
print(f" V^π({state}) = {value:.3f}")
print("\nImprovement:")
for state in mdp.states:
improvement = V_π_prime[state] - V_π[state]
symbol = "↑" if improvement > 0 else ("↓" if improvement < 0 else "→")
print(f" State {state}: {V_π[state]:.3f} → {V_π_prime[state]:.3f} ({symbol}{abs(improvement):.3f})")
return mdp, π, V_π, π_prime, V_π_primedef complete_policy_iteration_example():
"""Complete example of policy iteration"""
print("="*70)
print("COMPLETE POLICY ITERATION EXAMPLE")
print("="*70)
# Create a grid world MDP
class GridMDP:
def __init__(self, rows=3, cols=3):
self.rows = rows
self.cols = cols
self.states = [(r, c) for r in range(rows) for c in range(cols)]
# Actions: 0=up, 1=right, 2=down, 3=left
self.actions = [0, 1, 2, 3]
# Special states
self.goal = (rows-1, cols-1)
self.trap = (1, 1)
# Build transitions
self.transitions = self._build_transitions()
# Build rewards
self.rewards = self._build_rewards()
def _build_transitions(self):
transitions = {}
for state in self.states:
r, c = state
for action in self.actions:
next_state_probs = {}
if state == self.goal or state == self.trap:
# Terminal states
next_state_probs[state] = 1.0
else:
# Movement with 80% success, 10% left, 10% right of intended
if action == 0: # up
intended = (max(0, r-1), c)
left = (r, max(0, c-1))
right = (r, min(self.cols-1, c+1))
elif action == 1: # right
intended = (r, min(self.cols-1, c+1))
left = (max(0, r-1), c)
right = (min(self.rows-1, r+1), c)
elif action == 2: # down
intended = (min(self.rows-1, r+1), c)
left = (r, min(self.cols-1, c+1))
right = (r, max(0, c-1))
else: # left
intended = (r, max(0, c-1))
left = (min(self.rows-1, r+1), c)
right = (max(0, r-1), c)
next_state_probs[intended] = 0.8
next_state_probs[left] = 0.1
next_state_probs[right] = 0.1
transitions[(state, action)] = next_state_probs
return transitions
def _build_rewards(self):
rewards = {}
for (state, action), next_states in self.transitions.items():
for next_state, prob in next_states.items():
if next_state == self.goal:
reward = 10
elif next_state == self.trap:
reward = -10
else:
reward = -1 # Cost for each step
rewards[(state, action, next_state)] = reward
return rewards
def get_possible_actions(self, state):
return self.actions # All actions always available
def visualize(self, V=None, policy=None):
"""Visualize grid with values and/or policy"""
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(10, 8))
# Create grid
for r in range(self.rows):
for c in range(self.cols):
# Cell color
if (r, c) == self.goal:
color = 'lightgreen'
elif (r, c) == self.trap:
color = 'lightcoral'
else:
color = 'lightblue'
rect = plt.Rectangle((c-0.5, r-0.5), 1, 1,
facecolor=color, edgecolor='black', linewidth=2)
ax.add_patch(rect)
# Add value if provided
if V is not None and (r, c) in V:
value = V[(r, c)]
ax.text(c, r, f'{value:.1f}',
ha='center', va='center', fontsize=12, fontweight='bold')
# Add policy if provided
if policy is not None and (r, c) in policy:
action = list(policy[(r, c)].keys())[0] if policy[(r, c)] else None
if action is not None:
action_symbols = {0: '↑', 1: '→', 2: '↓', 3: '←'}
symbol = action_symbols.get(action, '?')
ax.text(c, r-0.3, symbol, ha='center', va='center',
fontsize=20, color='red')
ax.set_xlim(-0.5, self.cols-0.5)
ax.set_ylim(-0.5, self.rows-0.5)
ax.set_aspect('equal')
ax.invert_yaxis() # So (0,0) is at top-left
ax.set_xticks(range(self.cols))
ax.set_yticks(range(self.rows))
ax.set_xlabel('Column')
ax.set_ylabel('Row')
ax.set_title('Grid World MDP')
ax.grid(True, alpha=0.3)
# Legend
legend_elements = [
plt.Rectangle((0,0),1,1, facecolor='lightgreen', edgecolor='black', label='Goal (+10)'),
plt.Rectangle((0,0),1,1, facecolor='lightcoral', edgecolor='black', label='Trap (-10)'),
plt.Rectangle((0,0),1,1, facecolor='lightblue', edgecolor='black', label='Normal (-1 per step)')
]
ax.legend(handles=legend_elements, loc='upper right')
plt.tight_layout()
plt.show()
# Create MDP and DP solver
mdp = GridMDP(rows=3, cols=3)
dp = DynamicProgramming(mdp)
print("Grid World MDP created:")
print(f" States: {mdp.rows} × {mdp.cols} = {len(mdp.states)}")
print(f" Actions: {len(mdp.actions)}")
print(f" Goal: {mdp.goal} (reward +10)")
print(f" Trap: {mdp.trap} (reward -10)")
print(f" Step cost: -1")
print("\n" + "-"*70)
print("INITIAL VISUALIZATION")
print("-"*70)
mdp.visualize()
print("\n" + "-"*70)
print("RUNNING POLICY ITERATION")
print("-"*70)
# Run policy iteration
final_policy, final_V, policy_history, value_history = dp.policy_iteration(gamma=0.9)
print("\n" + "-"*70)
print("FINAL RESULTS")
print("-"*70)
print(f"\nOptimal Value Function:")
for r in range(mdp.rows):
row_vals = []
for c in range(mdp.cols):
val = final_V.get((r, c), 0)
row_vals.append(f"{val:6.2f}")
print(" " + " ".join(row_vals))
print(f"\nOptimal Policy (arrows show best action):")
mdp.visualize(V=final_V, policy=final_policy)
print("\n" + "-"*70)
print("POLICY EVOLUTION")
print("-"*70)
# Show how policy changed during iteration
action_symbols = {0: '↑', 1: '→', 2: '↓', 3: '←'}
for i, policy in enumerate(policy_history[:4]): # Show first 4 iterations
print(f"\nIteration {i}:")
for r in range(mdp.rows):
row_actions = []
for c in range(mdp.cols):
state = (r, c)
if state in policy and policy[state]:
action = list(policy[state].keys())[0]
row_actions.append(action_symbols.get(action, '?'))
else:
row_actions.append('·')
print(" " + " ".join(row_actions))
print("\n" + "-"*70)
print("VALUE CONVERGENCE")
print("-"*70)
# Plot convergence
import matplotlib.pyplot as plt
# Calculate max value change per iteration
max_deltas = []
for i in range(1, len(value_history)):
max_delta = 0
for state in mdp.states:
delta = abs(value_history[i][state] - value_history[i-1][state])
max_delta = max(max_delta, delta)
max_deltas.append(max_delta)
plt.figure(figsize=(10, 5))
plt.plot(max_deltas, 'b-o', linewidth=2, markersize=4)
plt.yscale('log')
plt.xlabel('Iteration')
plt.ylabel('Max Value Change (log scale)')
plt.title('Policy Iteration Convergence')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print(f"\nConverged in {len(policy_history)-1} policy iterations")
print(f"Final max value change: {max_deltas[-1]:.6f}")
return mdp, dp, final_policy, final_V
# Run the example
grid_mdp, dp_solver, optimal_policy, optimal_values = complete_policy_iteration_example()def value_iteration_deep_dive():
"""Deep dive into value iteration algorithm"""
print("="*70)
print("VALUE ITERATION DEEP DIVE")
print("="*70)
# Create a simple MDP for demonstration
class SimpleMDP:
def __init__(self):
self.states = ['A', 'B']
self.actions = ['stay', 'switch']
self.transitions = {
('A', 'stay'): {'A': 1.0},
('A', 'switch'): {'B': 1.0},
('B', 'stay'): {'B': 1.0},
('B', 'switch'): {'A': 1.0}
}
self.rewards = {
('A', 'stay', 'A'): 1,
('A', 'switch', 'B'): 3,
('B', 'stay', 'B'): 2,
('B', 'switch', 'A'): 0
}
def get_possible_actions(self, state):
return self.actions
mdp = SimpleMDP()
dp = DynamicProgramming(mdp)
gamma = 0.9
print("\nMDP Description:")
print(" States: A, B")
print(" Actions: stay, switch")
print(" Discount factor γ = 0.9")
print("\n" + "-"*70)
print("INITIALIZATION")
print("-"*70)
# Initialize value function
V = {state: 0 for state in mdp.states}
print(f"V₀ = {V}")
print("\n" + "-"*70)
print("ITERATION 1")
print("-"*70)
V1 = {}
for state in mdp.states:
print(f"\nState {state}:")
# Calculate Q-values for each action
q_values = {}
for action in mdp.actions:
q = 0
next_state_probs = mdp.transitions.get((state, action), {})
for next_state, prob in next_state_probs.items():
reward = mdp.rewards.get((state, action, next_state), 0)
q += prob * (reward + gamma * V[next_state])
q_values[action] = q
print(f" Q({state}, {action}) = {q:.3f}")
# Take maximum
best_action = max(q_values, key=q_values.get)
best_value = q_values[best_action]
V1[state] = best_value
print(f" max_a Q({state}, a) = {best_value:.3f} (action: {best_action})")
print(f"\nV₁ = {V1}")
print("\n" + "-"*70)
print("ITERATION 2")
print("-"*70)
V2 = {}
for state in mdp.states:
print(f"\nState {state}:")
q_values = {}
for action in mdp.actions:
q = 0
next_state_probs = mdp.transitions.get((state, action), {})
for next_state, prob in next_state_probs.items():
reward = mdp.rewards.get((state, action, next_state), 0)
q += prob * (reward + gamma * V1[next_state])
q_values[action] = q
print(f" Q({state}, {action}) = {q:.3f}")
best_action = max(q_values, key=q_values.get)
best_value = q_values[best_action]
V2[state] = best_value
print(f" max_a Q({state}, a) = {best_value:.3f} (action: {best_action})")
print(f"\nV₂ = {V2}")
print("\n" + "-"*70)
print("CONTINUING TO CONVERGENCE")
print("-"*70)
# Run full value iteration
optimal_policy, optimal_V, value_history, deltas = dp.value_iteration(gamma=0.9, theta=1e-6)
print(f"\nConverged after {len(value_history)} iterations")
print(f"Optimal value function: { {s: f'{v:.3f}' for s, v in optimal_V.items()} }")
print(f"\nOptimal policy:")
for state in mdp.states:
action = list(optimal_policy[state].keys())[0] if optimal_policy[state] else None
print(f" π*({state}) = {action}")
print("\n" + "-"*70)
print("MATHEMATICAL INSIGHTS")
print("-"*70)
print("\nThe Bellman Optimality Equation:")
print(" V*(s) = max_a Σ_s' P(s'|s,a) [R(s,a,s') + γV*(s')]")
print("\nValue iteration update rule:")
print(" V_{k+1}(s) = max_a Σ_s' P(s'|s,a) [R(s,a,s') + γV_k(s')]")
print("\nKey properties:")
print(" 1. Contraction mapping: Guarantees convergence")
print(" 2. Monotonic improvement: V_{k+1}(s) ≥ V_k(s) for all s")
print(" 3. Finds optimal value function V*")
print("\n" + "-"*70)
print("CONVERGENCE PLOT")
print("-"*70)
import matplotlib.pyplot as plt
plt.figure(figsize=(12, 5))
# Plot value evolution
plt.subplot(1, 2, 1)
iterations = range(len(value_history))
for state in mdp.states:
values = [vh[state] for vh in value_history]
plt.plot(iterations, values, 'o-', linewidth=2, markersize=4, label=f'State {state}')
plt.xlabel('Iteration')
plt.ylabel('State Value')
plt.title('Value Iteration Convergence')
plt.legend()
plt.grid(True, alpha=0.3)
# Plot deltas
plt.subplot(1, 2, 2)
plt.plot(range(1, len(deltas)+1), deltas, 'r-o', linewidth=2, markersize=4)
plt.yscale('log')
plt.xlabel('Iteration')
plt.ylabel('Max Delta (log scale)')
plt.title('Convergence Rate')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
return mdp, optimal_V, optimal_policy, value_history, deltasdef frozen_lake_dp_project():
"""Dynamic Programming solution for Frozen Lake environment"""
print("="*70)
print("FROZEN LAKE DYNAMIC PROGRAMMING PROJECT")
print("="*70)
try:
import gymnasium as gym
gym_available = True
except ImportError:
print("gymnasium not available. Installing...")
import subprocess
import sys
subprocess.check_call([sys.executable, "-m", "pip", "install", "gymnasium"])
import gymnasium as gym
gym_available = True
if not gym_available:
print("Could not import gymnasium. Using custom implementation.")
return
print("\n1. Creating Frozen Lake Environment...")
# Create environment
env = gym.make('FrozenLake-v1', map_name="4x4", is_slippery=True)
print(f" State space size: {env.observation_space.n}")
print(f" Action space size: {env.action_space.n}")
# Extract MDP from environment
print("\n2. Extracting MDP parameters...")
# Note: In practice, we'd need to estimate P and R from the environment
# For this demo, we'll create a simplified MDP
class FrozenLakeMDP:
def __init__(self, env):
self.env = env
self.n_states = env.observation_space.n
self.n_actions = env.action_space.n
# We'll estimate transition probabilities by sampling
print(" Estimating transition probabilities...")
self.transitions = self._estimate_transitions(samples_per_state=1000)
print(" Estimating reward function...")
self.rewards = self._estimate_rewards(samples_per_state=1000)
# States are just integers 0..n_states-1
self.states = list(range(self.n_states))
self.actions = list(range(self.n_actions))
def _estimate_transitions(self, samples_per_state=100):
"""Estimate transition probabilities by sampling"""
transitions = {}
for s in range(self.n_states):
for a in range(self.n_actions):
next_state_counts = {}
# Reset to specific state (requires custom reset)
# For simplicity, we'll sample from the environment
env_copy = gym.make('FrozenLake-v1', map_name="4x4", is_slippery=True)
# Sample transitions
for _ in range(samples_per_state):
# We need to set the state - in real implementation use env.unwrapped
# For now, we'll use a simpler approach
env_copy.reset()
# This is simplified - real implementation would use env.unwrapped.set_state()
pass
# Simplified: assume deterministic transitions for demo
# In real DP, you need the exact model
transitions[(s, a)] = {(s+1) % self.n_states: 1.0}
return transitions
def _estimate_rewards(self, samples_per_state=100):
"""Estimate reward function by sampling"""
rewards = {}
# Simplified: FrozenLake gives reward 1 for reaching goal, 0 otherwise
for s in range(self.n_states):
for a in range(self.n_actions):
for s_prime in range(self.n_states):
# Goal is state 15 in 4x4 map
if s_prime == 15:
reward = 1.0
else:
reward = 0.0
rewards[(s, a, s_prime)] = reward
return rewards
def get_possible_actions(self, state):
return self.actions
# Create MDP
mdp = FrozenLakeMDP(env)
print("\n3. Running Dynamic Programming...")
# Since we don't have exact model, we'll use a simpler approach
# In real DP, you need the exact transition probabilities
print(" Note: For exact DP solution, we need the complete transition model.")
print(" In practice, Frozen Lake is slippery (non-deterministic),")
print(" so we need to know the exact probabilities.")
print("\n Creating simplified deterministic version...")
# Create a deterministic version for demonstration
class DeterministicFrozenLake:
def __init__(self, size=4):
self.size = size
self.n_states = size * size
self.states = list(range(self.n_states))
self.actions = [0, 1, 2, 3] # 0: left, 1: down, 2: right, 3: up
# Build deterministic transitions
self.transitions = self._build_transitions()
self.rewards = self._build_rewards()
def _build_transitions(self):
transitions = {}
for s in self.states:
row = s // self.size
col = s % self.size
for a in self.actions:
next_state_probs = {}
if a == 0: # left
new_col = max(0, col - 1)
new_state = row * self.size + new_col
elif a == 1: # down
new_row = min(self.size - 1, row + 1)
new_state = new_row * self.size + col
elif a == 2: # right
new_col = min(self.size - 1, col + 1)
new_state = row * self.size + new_col
else: # up
new_row = max(0, row - 1)
new_state = new_row * self.size + col
next_state_probs[new_state] = 1.0
transitions[(s, a)] = next_state_probs
return transitions
def _build_rewards(self):
rewards = {}
goal_state = self.n_states - 1 # Bottom-right corner
hole_states = [5, 7, 11, 12] # Example hole positions for 4x4
for (s, a), next_states in self.transitions.items():
for s_prime, prob in next_states.items():
if s_prime == goal_state:
reward = 1.0
elif s_prime in hole_states:
reward = -1.0
else:
reward = 0.0
rewards[(s, a, s_prime)] = reward
return rewards
def get_possible_actions(self, state):
return self.actions
def visualize_policy(self, policy, title="Policy"):
"""Visualize policy on grid"""
import matplotlib.pyplot as plt
import numpy as np
action_symbols = {0: '←', 1: '↓', 2: '→', 3: '↑'}
grid = np.full((self.size, self.size), '', dtype=object)
for s in self.states:
row = s // self.size
col = s % self.size
if s in policy and policy[s]:
action = list(policy[s].keys())[0]
grid[row, col] = action_symbols.get(action, '?')
else:
grid[row, col] = '·'
# Mark goal and holes
goal_state = self.n_states - 1
goal_row, goal_col = goal_state // self.size, goal_state % self.size
grid[goal_row, goal_col] = 'G'
hole_states = [5, 7, 11, 12]
for hs in hole_states:
hr, hc = hs // self.size, hs % self.size
grid[hr, hc] = 'H'
# Start state
grid[0, 0] = 'S'
# Plot
fig, ax = plt.subplots(figsize=(8, 8))
# Create grid lines
for i in range(self.size + 1):
ax.axhline(i - 0.5, color='black', linewidth=2)
ax.axvline(i - 0.5, color='black', linewidth=2)
# Add text
for i in range(self.size):
for j in range(self.size):
ax.text(j, i, grid[i, j], ha='center', va='center',
fontsize=20, fontweight='bold')
ax.set_xlim(-0.5, self.size - 0.5)
ax.set_ylim(-0.5, self.size - 0.5)
ax.invert_yaxis()
ax.set_xticks([])
ax.set_yticks([])
ax.set_title(title)
# Add legend
legend_text = "S: Start, G: Goal, H: Hole, Arrows: Optimal action"
ax.text(0.5, -0.1, legend_text, transform=ax.transAxes,
ha='center', fontsize=10)
plt.tight_layout()
plt.show()
print("\n4. Creating and solving deterministic version...")
det_mdp = DeterministicFrozenLake(size=4)
dp = DynamicProgramming(det_mdp)
# Run value iteration
optimal_policy, optimal_V, value_history, deltas = dp.value_iteration(gamma=0.9)
print(f"\n5. Results:")
print(f" Number of iterations: {len(value_history)}")
print(f" Optimal value at start state: {optimal_V.get(0, 0):.3f}")
print("\n6. Visualizing optimal policy...")
det_mdp.visualize_policy(optimal_policy, title="Optimal Policy (Deterministic)")
print("\n7. Testing policy in actual environment...")
# Test in stochastic environment
def test_policy_in_env(policy, n_episodes=100):
env = gym.make('FrozenLake-v1', map_name="4x4", is_slippery=True, render_mode=None)
successes = 0
total_rewards = []
for episode in range(n_episodes):
state, _ = env.reset()
total_reward = 0
done = False
while not done:
# Get action from policy
if state in policy and policy[state]:
action = list(policy[state].keys())[0]
else:
action = env.action_space.sample()
# Take action
state, reward, terminated, truncated, _ = env.step(action)
done = terminated or truncated
total_reward += reward
if reward == 1.0: # Reached goal
successes += 1
total_rewards.append(total_reward)
success_rate = successes / n_episodes
avg_reward = sum(total_rewards) / n_episodes
print(f" Success rate: {success_rate:.1%} ({successes}/{n_episodes})")
print(f" Average reward: {avg_reward:.3f}")
return success_rate, avg_reward
print("\n Testing deterministic policy in slippery environment:")
success_rate, avg_reward = test_policy_in_env(optimal_policy, n_episodes=1000)
print("\n" + "="*70)
print("KEY INSIGHTS")
print("="*70)
print("\n1. Dynamic Programming requires a complete model:")
print(" - Need exact P(s'|s,a) and R(s,a,s')")
print(" - Not practical for unknown environments")
print("\n2. Value Iteration vs Policy Iteration:")
print(" - Value iteration: Updates values directly")
print(" - Policy iteration: Alternates evaluation and improvement")
print(" - Both converge to optimal solution with complete model")
print("\n3. Limitations in practice:")
print(" - Need to know environment dynamics exactly")
print(" - Doesn't work for large state spaces (curse of dimensionality)")
print(" - Deterministic solutions may fail in stochastic environments")
print("\n" + "="*70)
print("NEXT STEPS")
print("="*70)
print("\nIn next chapters, we'll learn model-free methods:")
print("1. Monte Carlo: Learn from complete episodes")
print("2. Temporal Difference: Learn from incomplete episodes")
print("3. Q-learning: Learn optimal policy without model")
return det_mdp, optimal_policy, optimal_V, success_rate
# Run the project
if __name__ == "__main__":
frozen_lake_mdp, policy, values, success = frozen_lake_dp_project()Chapter 4 Key Takeaways:
- Dynamic Programming requires complete knowledge of the environment
- Policy Iteration alternates between evaluation and improvement
- Value Iteration directly updates the value function
- Both methods guarantee convergence to optimal policy
- DP is powerful for planning but impractical for large/unknown environments
Continuing from Chapter 4...
Analogy: Learning to Bake Without a Recipe Imagine learning to bake cookies through trial and error:
- DP approach: Read all baking theory first, calculate exact oven times
- MC approach: Actually bake cookies, taste them, learn what works
Monte Carlo Principle: Learn from complete episodes of experience.
import numpy as np
import matplotlib.pyplot as plt
from collections import defaultdict
class MonteCarloLearning:
"""Monte Carlo methods for RL"""
def __init__(self, env, gamma=0.9):
self.env = env
self.gamma = gamma
# Initialize value function and returns
self.V = defaultdict(float) # State values
self.returns = defaultdict(list) # Returns for each state
def generate_episode(self, policy):
"""Generate an episode following policy"""
episode = []
state = self.env.reset()
done = False
while not done:
# Get action from policy
if isinstance(policy, dict):
# Deterministic policy
action = policy[state]
else:
# Stochastic policy function
action = policy(state)
# Take action
next_state, reward, done = self.env.step(action)
# Store transition
episode.append((state, action, reward))
state = next_state
return episode
def first_visit_mc_prediction(self, policy, num_episodes=1000):
"""
First-visit MC prediction
Average returns following first visit to each state
"""
for episode_num in range(num_episodes):
# Generate episode
episode = self.generate_episode(policy)
states_in_episode = [step[0] for step in episode]
rewards = [step[2] for step in episode]
# Calculate returns
G = 0
visited_states = set()
# Work backwards through episode
for t in reversed(range(len(episode))):
state, action, reward = episode[t]
# Update return
G = self.gamma * G + reward
# First-visit: only update if state not visited earlier in episode
if state not in visited_states:
visited_states.add(state)
self.returns[state].append(G)
self.V[state] = np.mean(self.returns[state])
if (episode_num + 1) % 100 == 0:
print(f"Episode {episode_num + 1}/{num_episodes}")
return self.V
def every_visit_mc_prediction(self, policy, num_episodes=1000):
"""
Every-visit MC prediction
Average returns following every visit to each state
"""
for episode_num in range(num_episodes):
episode = self.generate_episode(policy)
rewards = [step[2] for step in episode]
# Calculate returns
G = 0
# Work backwards
for t in reversed(range(len(episode))):
state, action, reward = episode[t]
# Update return
G = self.gamma * G + reward
# Every-visit: update on every visit
self.returns[state].append(G)
self.V[state] = np.mean(self.returns[state])
return self.V
def mc_exploring_starts(self, num_episodes=10000, epsilon=0.1):
"""
Monte Carlo with Exploring Starts
For control (finding optimal policy)
"""
# Initialize Q-function and policy
Q = defaultdict(lambda: np.zeros(self.env.action_space.n))
policy = defaultdict(lambda: np.random.randint(self.env.action_space.n))
# Keep track of returns
returns = defaultdict(list)
for episode_num in range(num_episodes):
# Exploring start: random initial state-action pair
state = self.env.reset()
# Generate episode with exploring starts
episode = []
done = False
while not done:
# Epsilon-greedy action selection
if np.random.random() < epsilon:
action = np.random.randint(self.env.action_space.n)
else:
action = np.argmax(Q[state])
next_state, reward, done = self.env.step(action)
episode.append((state, action, reward))
state = next_state
# Process episode
G = 0
visited_state_actions = set()
# Work backwards through episode
for t in reversed(range(len(episode))):
state, action, reward = episode[t]
G = self.gamma * G + reward
# First-visit for state-action pairs
if (state, action) not in visited_state_actions:
visited_state_actions.add((state, action))
returns[(state, action)].append(G)
Q[state][action] = np.mean(returns[(state, action)])
# Update policy (greedy)
policy[state] = np.argmax(Q[state])
return policy, QFirst-Visit vs Every-Visit MC:
def compare_mc_methods():
"""Compare first-visit and every-visit MC"""
# Simple environment
class SimpleEnv:
def __init__(self):
self.states = ['A', 'B', 'C', 'Terminal']
self.current_state = 'A'
def reset(self):
self.current_state = 'A'
return self.current_state
def step(self, action):
"""Simple deterministic transitions"""
transitions = {
'A': {'right': 'B', 'down': 'C'},
'B': {'right': 'Terminal', 'down': 'C'},
'C': {'right': 'Terminal', 'up': 'A'},
'Terminal': {}
}
if self.current_state == 'Terminal':
return self.current_state, 0, True
# Simple deterministic policy: always go right
action = 'right'
next_state = transitions[self.current_state][action]
# Rewards
rewards = {
('A', 'right', 'B'): 1,
('B', 'right', 'Terminal'): 10,
('C', 'right', 'Terminal'): 5
}
reward = rewards.get((self.current_state, action, next_state), 0)
self.current_state = next_state
done = (next_state == 'Terminal')
return next_state, reward, done
env = SimpleEnv()
mc = MonteCarloLearning(env, gamma=0.9)
# Fixed policy: always go right
policy = {'A': 'right', 'B': 'right', 'C': 'right'}
print("="*60)
print("COMPARING MONTE CARLO PREDICTION METHODS")
print("="*60)
# First-visit MC
print("\n1. First-Visit MC Prediction:")
V_first_visit = mc.first_visit_mc_prediction(policy, num_episodes=1000)
print(f" V(A) = {V_first_visit.get('A', 0):.3f}")
print(f" V(B) = {V_first_visit.get('B', 0):.3f}")
print(f" V(C) = {V_first_visit.get('C', 0):.3f}")
# Reset for every-visit
mc.V = defaultdict(float)
mc.returns = defaultdict(list)
# Every-visit MC
print("\n2. Every-Visit MC Prediction:")
V_every_visit = mc.every_visit_mc_prediction(policy, num_episodes=1000)
print(f" V(A) = {V_every_visit.get('A', 0):.3f}")
print(f" V(B) = {V_every_visit.get('B', 0):.3f}")
print(f" V(C) = {V_every_visit.get('C', 0):.3f}")
# Theoretical values (for comparison)
print("\n3. Theoretical Values (for comparison):")
print(" From A: A→B (1) then B→Terminal (10)")
print(" Return = 1 + 0.9×10 = 10.0")
print(" From B: B→Terminal (10)")
print(" Return = 10.0")
print(" From C: C→Terminal (5)")
print(" Return = 5.0")
return V_first_visit, V_every_visit
# Run comparison
V_first, V_every = compare_mc_methods()On-Policy vs Off-Policy Learning:
class MonteCarloControl:
"""Monte Carlo control methods"""
def __init__(self, env, gamma=0.9):
self.env = env
self.gamma = gamma
self.n_actions = env.action_space.n
def create_epsilon_greedy_policy(self, Q, epsilon):
"""Create epsilon-greedy policy based on Q-values"""
def policy(state):
if np.random.random() < epsilon:
# Explore: random action
return np.random.randint(self.n_actions)
else:
# Exploit: best action
return np.argmax(Q[state])
return policy
def on_policy_mc_control(self, num_episodes=10000, epsilon=0.1):
"""
On-policy MC control with epsilon-greedy policy
"""
# Initialize
Q = defaultdict(lambda: np.zeros(self.n_actions))
returns = defaultdict(list)
# Create initial policy
policy = self.create_epsilon_greedy_policy(Q, epsilon)
# Track performance
rewards_history = []
for episode_num in range(num_episodes):
# Generate episode
episode = []
state = self.env.reset()
done = False
while not done:
action = policy(state)
next_state, reward, done = self.env.step(action)
episode.append((state, action, reward))
state = next_state
# Calculate returns and update Q-values
G = 0
visited_state_actions = set()
# Work backwards through episode
for t in reversed(range(len(episode))):
state, action, reward = episode[t]
G = self.gamma * G + reward
# First-visit MC
if (state, action) not in visited_state_actions:
visited_state_actions.add((state, action))
returns[(state, action)].append(G)
Q[state][action] = np.mean(returns[(state, action)])
# Track performance
total_reward = sum([step[2] for step in episode])
rewards_history.append(total_reward)
# Decay epsilon over time
epsilon = max(0.01, epsilon * 0.995)
if (episode_num + 1) % 1000 == 0:
avg_reward = np.mean(rewards_history[-1000:])
print(f"Episode {episode_num + 1}: Avg reward (last 1000) = {avg_reward:.2f}")
# Extract deterministic policy from Q-values
deterministic_policy = {}
for state in Q:
deterministic_policy[state] = np.argmax(Q[state])
return deterministic_policy, Q, rewards_history
def off_policy_mc_control(self, num_episodes=10000, epsilon=0.1):
"""
Off-policy MC control using importance sampling
Target policy: greedy
Behavior policy: epsilon-greedy
"""
# Initialize Q-function
Q = defaultdict(lambda: np.zeros(self.n_actions))
C = defaultdict(float) # Cumulative sum of weights
# Behavior policy (epsilon-greedy)
def behavior_policy(state):
if np.random.random() < epsilon:
return np.random.randint(self.n_actions)
else:
return np.argmax(Q[state])
# Target policy (greedy)
def target_policy(state):
return np.argmax(Q[state])
rewards_history = []
for episode_num in range(num_episodes):
# Generate episode using behavior policy
episode = []
state = self.env.reset()
done = False
while not done:
action = behavior_policy(state)
next_state, reward, done = self.env.step(action)
episode.append((state, action, reward))
state = next_state
# Calculate importance sampling weights
G = 0
W = 1.0 # Importance sampling ratio
# Work backwards through episode
for t in reversed(range(len(episode))):
state, action, reward = episode[t]
G = self.gamma * G + reward
# Update cumulative sum
C[(state, action)] += W
# Update Q-value
Q[state][action] += (W / C[(state, action)]) * (G - Q[state][action])
# If action not greedy under target policy, break
if action != target_policy(state):
break
# Update importance sampling ratio
# For greedy target policy: probability = 1
# For epsilon-greedy behavior policy:
# P(action|state) = 1 - epsilon + epsilon/n_actions for greedy action
behavior_prob = 1 - epsilon + epsilon/self.n_actions
W = W * (1.0 / behavior_prob)
# Track performance
total_reward = sum([step[2] for step in episode])
rewards_history.append(total_reward)
# Decay epsilon
epsilon = max(0.01, epsilon * 0.995)
# Extract deterministic policy
deterministic_policy = {}
for state in Q:
deterministic_policy[state] = np.argmax(Q[state])
return deterministic_policy, Q, rewards_historyclass BlackjackEnvironment:
"""Blackjack environment for Monte Carlo learning"""
def __init__(self):
self.reset()
def reset(self):
"""Reset game"""
self.player_hand = []
self.dealer_hand = []
# Deal initial cards
self.player_hand.append(self._draw_card())
self.dealer_hand.append(self._draw_card())
self.player_hand.append(self._draw_card())
self.dealer_hand.append(self._draw_card())
self.done = False
return self._get_state()
def _draw_card(self):
"""Draw a card (1-13, face cards = 10)"""
card = min(10, np.random.randint(1, 14))
return card
def _get_state(self):
"""Get current state"""
player_sum = sum(self.player_hand)
usable_ace = 1 in self.player_hand and player_sum <= 11
dealer_card = self.dealer_hand[0] # Only first card is visible
return (player_sum, usable_ace, dealer_card)
def step(self, action):
"""
Take action
0: stick (stop)
1: hit (draw another card)
"""
if action == 0: # stick
self.done = True
# Dealer's turn
while sum(self.dealer_hand) < 17:
self.dealer_hand.append(self._draw_card())
# Determine winner
player_sum = sum(self.player_hand)
dealer_sum = sum(self.dealer_hand)
if player_sum > 21: # Player bust
reward = -1
elif dealer_sum > 21: # Dealer bust
reward = 1
elif player_sum > dealer_sum: # Player wins
reward = 1
elif player_sum < dealer_sum: # Dealer wins
reward = -1
else: # Tie
reward = 0
else: # hit
self.player_hand.append(self._draw_card())
player_sum = sum(self.player_hand)
if player_sum > 21: # Bust
self.done = True
reward = -1
else:
reward = 0
return self._get_state(), reward, self.done
def get_possible_actions(self):
"""Get valid actions"""
return [0, 1] # Always can stick or hit
def simulate_episode(self, policy):
"""Simulate one episode following policy"""
state = self.reset()
episode = []
done = False
while not done:
# Get action from policy
if isinstance(policy, dict):
action = policy.get(state, 0) # Default to stick
else:
action = policy(state)
# Take action
next_state, reward, done = self.step(action)
episode.append((state, action, reward))
state = next_state
return episode
def blackjack_monte_carlo_experiment():
"""Complete Blackjack MC experiment"""
print("="*70)
print("BLACKJACK MONTE CARLO EXPERIMENT")
print("="*70)
# Create environment
env = BlackjackEnvironment()
print("\n1. Initializing Monte Carlo Learning...")
mc = MonteCarloLearning(env, gamma=1.0) # No discount for episodic tasks
# Define a simple policy: stick on 20 or 21, otherwise hit
def simple_policy(state):
player_sum, usable_ace, dealer_card = state
if player_sum >= 20:
return 0 # stick
else:
return 1 # hit
print("\n2. Running First-Visit MC Prediction...")
V = mc.first_visit_mc_prediction(simple_policy, num_episodes=50000)
print("\n3. Value Function Results:")
# Extract values for visualization
player_sums = list(range(12, 22))
dealer_cards = list(range(1, 11))
# Create heatmap data
data_no_ace = np.zeros((len(player_sums), len(dealer_cards)))
data_ace = np.zeros((len(player_sums), len(dealer_cards)))
for i, p_sum in enumerate(player_sums):
for j, d_card in enumerate(dealer_cards):
# Without usable ace
state = (p_sum, False, d_card)
data_no_ace[i, j] = V.get(state, 0)
# With usable ace
state = (p_sum, True, d_card)
data_ace[i, j] = V.get(state, 0)
# Plot heatmaps
fig, axes = plt.subplots(1, 2, figsize=(15, 6))
# No usable ace
im1 = axes[0].imshow(data_no_ace, cmap='RdYlGn', aspect='auto')
axes[0].set_title('Value Function: No Usable Ace')
axes[0].set_xlabel('Dealer Showing')
axes[0].set_ylabel('Player Sum')
axes[0].set_xticks(range(len(dealer_cards)))
axes[0].set_xticklabels(dealer_cards)
axes[0].set_yticks(range(len(player_sums)))
axes[0].set_yticklabels(player_sums)
plt.colorbar(im1, ax=axes[0])
# Add value annotations
for i in range(len(player_sums)):
for j in range(len(dealer_cards)):
axes[0].text(j, i, f'{data_no_ace[i, j]:.1f}',
ha='center', va='center', color='black')
# With usable ace
im2 = axes[1].imshow(data_ace, cmap='RdYlGn', aspect='auto')
axes[1].set_title('Value Function: With Usable Ace')
axes[1].set_xlabel('Dealer Showing')
axes[1].set_ylabel('Player Sum')
axes[1].set_xticks(range(len(dealer_cards)))
axes[1].set_xticklabels(dealer_cards)
axes[1].set_yticks(range(len(player_sums)))
axes[1].set_yticklabels(player_sums)
plt.colorbar(im2, ax=axes[1])
# Add value annotations
for i in range(len(player_sums)):
for j in range(len(dealer_cards)):
axes[1].text(j, i, f'{data_ace[i, j]:.1f}',
ha='center', va='center', color='black')
plt.tight_layout()
plt.show()
print("\n4. Finding Optimal Policy with MC Control...")
# Convert environment to standard interface
class StandardBlackjack:
def __init__(self):
self.env = BlackjackEnvironment()
self.action_space = type('ActionSpace', (), {'n': 2})()
self.observation_space = None
def reset(self):
return self.env.reset()
def step(self, action):
return self.env.step(action)
std_env = StandardBlackjack()
mc_control = MonteCarloControl(std_env, gamma=1.0)
print(" Running on-policy MC control...")
policy, Q, rewards = mc_control.on_policy_mc_control(num_episodes=100000, epsilon=0.1)
# Extract optimal policy for visualization
optimal_policy_no_ace = np.zeros((len(player_sums), len(dealer_cards)))
optimal_policy_ace = np.zeros((len(player_sums), len(dealer_cards)))
for i, p_sum in enumerate(player_sums):
for j, d_card in enumerate(dealer_cards):
# Without usable ace
state = (p_sum, False, d_card)
optimal_policy_no_ace[i, j] = policy.get(state, 0)
# With usable ace
state = (p_sum, True, d_card)
optimal_policy_ace[i, j] = policy.get(state, 0)
# Plot optimal policies
fig, axes = plt.subplots(1, 2, figsize=(15, 6))
# No usable ace
im1 = axes[0].imshow(optimal_policy_no_ace, cmap='coolwarm', aspect='auto', vmin=0, vmax=1)
axes[0].set_title('Optimal Policy: No Usable Ace (0=Stick, 1=Hit)')
axes[0].set_xlabel('Dealer Showing')
axes[0].set_ylabel('Player Sum')
axes[0].set_xticks(range(len(dealer_cards)))
axes[0].set_xticklabels(dealer_cards)
axes[0].set_yticks(range(len(player_sums)))
axes[0].set_yticklabels(player_sums)
plt.colorbar(im1, ax=axes[0], ticks=[0, 1])
# With usable ace
im2 = axes[1].imshow(optimal_policy_ace, cmap='coolwarm', aspect='auto', vmin=0, vmax=1)
axes[1].set_title('Optimal Policy: With Usable Ace (0=Stick, 1=Hit)')
axes[1].set_xlabel('Dealer Showing')
axes[1].set_ylabel('Player Sum')
axes[1].set_xticks(range(len(dealer_cards)))
axes[1].set_xticklabels(dealer_cards)
axes[1].set_yticks(range(len(player_sums)))
axes[1].set_yticklabels(player_sums)
plt.colorbar(im2, ax=axes[1], ticks=[0, 1])
plt.tight_layout()
plt.show()
print("\n5. Performance Analysis:")
print(f" Final 1000 episodes average reward: {np.mean(rewards[-1000:]):.3f}")
print(f" Best single episode reward: {np.max(rewards)}")
# Plot learning curve
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.plot(rewards, alpha=0.5)
plt.xlabel('Episode')
plt.ylabel('Reward')
plt.title('Learning Curve (Raw)')
plt.grid(True, alpha=0.3)
plt.subplot(1, 2, 2)
window_size = 1000
moving_avg = np.convolve(rewards, np.ones(window_size)/window_size, mode='valid')
plt.plot(moving_avg)
plt.xlabel('Episode')
plt.ylabel('Average Reward (1000-episode window)')
plt.title('Smoothed Learning Curve')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\n" + "="*70)
print("KEY INSIGHTS FROM BLACKJACK")
print("="*70)
print("\n1. With usable ace:")
print(" - Can hit more aggressively (ace can be 1 or 11)")
print(" - Optimal to hit on lower sums")
print("\n2. Without usable ace:")
print(" - More conservative strategy")
print(" - Stick earlier to avoid busting")
print("\n3. Against high dealer cards (7-10):")
print(" - More aggressive hitting")
print(" - Need higher total to compete")
print("\n4. Monte Carlo strengths:")
print(" - Learns from actual experience")
print(" - No model needed")
print(" - Handles stochastic environments")
return env, V, policy, Q, rewards
# Run the Blackjack experiment
blackjack_env, V_values, optimal_policy, Q_values, reward_history = blackjack_monte_carlo_experiment()def importance_sampling_demo():
"""Demonstrate importance sampling for off-policy MC"""
print("="*70)
print("IMPORTANCE SAMPLING DEMONSTRATION")
print("="*70)
# Simple environment with two actions
class SimpleImportanceEnv:
def __init__(self):
self.states = ['s1', 's2', 's3']
self.actions = [0, 1] # 0=left, 1=right
def reset(self):
self.state = 's1'
return self.state
def step(self, action):
transitions = {
('s1', 0): ('s2', 1), # left gives +1
('s1', 1): ('s3', 10), # right gives +10 (better!)
('s2', 0): ('s3', 2),
('s2', 1): ('s1', 1),
('s3', 0): ('s1', 0),
('s3', 1): ('s2', 0)
}
if (self.state, action) in transitions:
next_state, reward = transitions[(self.state, action)]
else:
next_state, reward = self.state, 0
self.state = next_state
done = False
return next_state, reward, done
env = SimpleImportanceEnv()
print("\n1. Understanding the Problem:")
print(" Target policy π: always choose action 1 (right)")
print(" Behavior policy b: choose action 0 with 0.8 prob, action 1 with 0.2 prob")
print(" We want to learn Q^π from episodes generated by b")
print("\n2. Importance Sampling Ratio:")
print(" Weight = Π_{k=t}^{T-1} [π(A_k|S_k) / b(A_k|S_k)]")
print(" For deterministic π and stochastic b:")
print(" - If π would choose same action as b: weight increases")
print(" - If π would choose different action: weight = 0")
print("\n3. Implementation:")
# Target policy (deterministic, always choose action 1)
def target_policy(state):
return 1
# Behavior policy (stochastic)
def behavior_policy(state):
return np.random.choice([0, 1], p=[0.8, 0.2])
# Generate episode with behavior policy
def generate_episode_with_policy(policy):
episode = []
state = env.reset()
done = False
while not done and len(episode) < 10: # Limit steps
action = policy(state)
next_state, reward, done = env.step(action)
episode.append((state, action, reward))
state = next_state
return episode
print("\n4. Example Episode from Behavior Policy:")
episode = generate_episode_with_policy(behavior_policy)
for i, (state, action, reward) in enumerate(episode):
print(f" Step {i}: State={state}, Action={action}, Reward={reward}")
print("\n5. Calculating Importance Sampling Weights:")
G = 0
weights = []
cumulative_weights = []
# Work backwards
for t in reversed(range(len(episode))):
state, action, reward = episode[t]
G = 0.9 * G + reward # gamma = 0.9
# Importance sampling ratio for this step
if action == target_policy(state):
# π(action|state) = 1 (deterministic)
# b(action|state) = 0.2 for action 1, 0.8 for action 0
behavior_prob = 0.2 if action == 1 else 0.8
weight = 1.0 / behavior_prob
else:
weight = 0 # π would never take this action
weights.insert(0, weight)
cumulative_weights.insert(0, weight * (cumulative_weights[0] if cumulative_weights else 1))
print("\n6. Results:")
print(" Episode return G = {:.2f}".format(G))
print(" Step-by-step weights:")
for i, (state, action, reward) in enumerate(episode):
print(f" Step {i}: State={state}, Action={action}, "
f"Weight={weights[i]:.2f}, CumWeight={cumulative_weights[i]:.2f}")
print("\n7. Weighted Returns:")
weighted_return = G * cumulative_weights[-1]
print(f" Weighted return = {G:.2f} × {cumulative_weights[-1]:.2f} = {weighted_return:.2f}")
print("\n8. Ordinary Importance Sampling vs Weighted:")
print(" Ordinary IS: average of weighted returns")
print(" Weighted IS: weighted average of returns")
print(" Weighted IS has lower variance but is biased")
return episode, weights, cumulative_weights
# Run importance sampling demo
episode_data, is_weights, cum_weights = importance_sampling_demo()def windy_gridworld_project():
"""MC methods on Windy Gridworld"""
print("="*70)
print("WINDY GRIDWORLD PROJECT")
print("="*70)
class WindyGridworld:
"""Gridworld with wind that pushes agent upward"""
def __init__(self, width=10, height=7):
self.width = width
self.height = height
# Start and goal positions
self.start = (3, 0)
self.goal = (3, 7)
# Wind strength for each column
self.wind = [0, 0, 0, 1, 1, 1, 2, 2, 1, 0]
# Actions: 0=up, 1=right, 2=down, 3=left
self.actions = [0, 1, 2, 3]
self.action_deltas = {
0: (-1, 0), # up
1: (0, 1), # right
2: (1, 0), # down
3: (0, -1) # left
}
self.reset()
def reset(self):
"""Reset to start position"""
self.position = list(self.start)
self.steps = 0
self.done = False
return tuple(self.position)
def step(self, action):
"""Take action, considering wind"""
if self.done:
return tuple(self.position), 0, True
self.steps += 1
# Move according to action
dr, dc = self.action_deltas[action]
new_r = self.position[0] + dr
new_c = self.position[1] + dc
# Apply wind
wind_strength = self.wind[self.position[1]]
new_r -= wind_strength # Wind blows upward
# Clip to grid bounds
new_r = max(0, min(self.height - 1, new_r))
new_c = max(0, min(self.width - 1, new_c))
self.position = [new_r, new_c]
# Check if reached goal
if tuple(self.position) == self.goal:
self.done = True
reward = 0 # No penalty for reaching goal
else:
reward = -1 # Penalty for each step
# Limit episode length
if self.steps >= 100:
self.done = True
return tuple(self.position), reward, self.done
def render(self):
"""Render current state"""
grid = [['.' for _ in range(self.width)] for _ in range(self.height)]
# Mark start and goal
grid[self.start[0]][self.start[1]] = 'S'
grid[self.goal[0]][self.goal[1]] = 'G'
# Mark agent
r, c = self.position
grid[r][c] = 'A'
# Print with wind indicators
print("Windy Gridworld (wind blows ↑, strength shown below):")
print("+" + "-" * (2*self.width + 1) + "+")
for r in range(self.height):
print("| " + " ".join(grid[r]) + " |")
print("+" + "-" * (2*self.width + 1) + "+")
print("Wind: " + " ".join(str(w) for w in self.wind))
print(f"Steps: {self.steps}, Position: {self.position}")
print("\n1. Creating Windy Gridworld Environment...")
env = WindyGridworld()
print("\n2. Testing Environment...")
state = env.reset()
env.render()
# Test random actions
print("\n3. Testing Random Policy for 5 steps:")
for i in range(5):
action = np.random.choice(env.actions)
state, reward, done = env.step(action)
print(f" Step {i+1}: Action={action}, Reward={reward}, New state={state}")
if done:
break
print("\n4. Implementing Monte Carlo Control...")
# Convert to standard interface
class StandardWindyGridworld:
def __init__(self):
self.env = WindyGridworld()
self.action_space = type('ActionSpace', (), {'n': 4})()
self.states = [(r, c) for r in range(env.height) for c in range(env.width)]
def reset(self):
return self.env.reset()
def step(self, action):
return self.env.step(action)
std_env = StandardWindyGridworld()
mc_control = MonteCarloControl(std_env, gamma=0.9)
print("\n5. Training with On-Policy MC Control...")
policy, Q, rewards = mc_control.on_policy_mc_control(
num_episodes=2000, epsilon=0.1
)
print("\n6. Visualizing Learned Policy...")
# Extract policy for visualization
policy_grid = np.full((env.height, env.width), ' ', dtype=object)
action_symbols = {0: '↑', 1: '→', 2: '↓', 3: '←'}
for r in range(env.height):
for c in range(env.width):
state = (r, c)
if state in policy:
action = policy[state]
policy_grid[r, c] = action_symbols.get(action, '?')
else:
policy_grid[r, c] = '.'
# Mark special positions
policy_grid[env.start] = 'S'
policy_grid[env.goal] = 'G'
# Plot
fig, axes = plt.subplots(1, 2, figsize=(15, 6))
# Policy visualization
ax1 = axes[0]
im1 = ax1.imshow(np.zeros((env.height, env.width)), cmap='Blues', alpha=0.3)
# Add wind indicators
for c in range(env.width):
wind_str = str(env.wind[c])
ax1.text(c, env.height + 0.5, wind_str, ha='center', va='center', fontweight='bold')
# Add policy arrows
for r in range(env.height):
for c in range(env.width):
ax1.text(c, r, policy_grid[r, c], ha='center', va='center',
fontsize=12, fontweight='bold')
ax1.set_title('Learned Policy (Arrows show optimal action)')
ax1.set_xlabel('Column')
ax1.set_ylabel('Row')
ax1.set_xticks(range(env.width))
ax1.set_yticks(range(env.height))
ax1.grid(True, alpha=0.3)
# Learning curve
ax2 = axes[1]
window_size = 50
moving_avg = np.convolve(rewards, np.ones(window_size)/window_size, mode='valid')
ax2.plot(moving_avg, linewidth=2)
ax2.set_xlabel('Episode')
ax2.set_ylabel('Average Reward (50-episode window)')
ax2.set_title('Learning Curve')
ax2.grid(True, alpha=0.3)
# Add horizontal line at optimal (shortest path ~15 steps = -15 reward)
optimal_reward = -15
ax2.axhline(y=optimal_reward, color='r', linestyle='--', alpha=0.5,
label=f'Optimal (~{optimal_reward})')
ax2.legend()
plt.tight_layout()
plt.show()
print("\n7. Testing Learned Policy...")
def test_policy(policy, num_tests=10):
"""Test policy on environment"""
test_rewards = []
steps_to_goal = []
for test in range(num_tests):
state = env.reset()
total_reward = 0
steps = 0
done = False
while not done:
if state in policy:
action = policy[state]
else:
action = np.random.choice(env.actions)
state, reward, done = env.step(action)
total_reward += reward
steps += 1
test_rewards.append(total_reward)
steps_to_goal.append(steps)
return test_rewards, steps_to_goal
test_rewards, test_steps = test_policy(policy, num_tests=100)
print(f" Average reward: {np.mean(test_rewards):.2f}")
print(f" Average steps to goal: {np.mean(test_steps):.2f}")
print(f" Best performance: {np.max(test_rewards)} reward, {np.min(test_steps)} steps")
print(f" Worst performance: {np.min(test_rewards)} reward, {np.max(test_steps)} steps")
print("\n8. Comparing with Optimal Path:")
print(" Without wind, optimal path: 7 steps right = -7 reward")
print(" With wind, need to account for upward drift")
print(" Optimal strategy: move down in windy columns to compensate")
print("\n" + "="*70)
print("MONTE CARLO SUMMARY")
print("="*70)
print("\nAdvantages of Monte Carlo:")
print("1. Model-free: Doesn't need P(s'|s,a)")
print("2. Learns from actual experience")
print("3. Handles stochastic policies naturally")
print("4. Simple to implement")
print("\nDisadvantages:")
print("1. High variance (learns from complete episodes)")
print("2. Must wait until end of episode to update")
print("3. Exploring starts can be impractical")
print("\nKey Concepts:")
print("- First-visit vs every-visit MC")
print("- Exploring starts for control")
print("- Importance sampling for off-policy learning")
print("- ε-greedy policies for exploration")
return env, policy, Q, rewards
# Run Windy Gridworld project
windy_env, windy_policy, windy_Q, windy_rewards = windy_gridworld_project()Chapter 5 Key Takeaways:
- Monte Carlo methods learn from complete episodes
- No model required - learn directly from experience
- First-visit MC averages returns after first visit to each state
- Every-visit MC averages returns after every visit
- On-policy methods learn the policy they're following
- Off-policy methods learn about a target policy from a behavior policy
- Importance sampling enables off-policy learning
- ε-greedy policies balance exploration and exploitation
Analogy: Predicting the Weather Imagine predicting today's high temperature:
- Monte Carlo: Wait until midnight, then update based on actual high
- Temporal Difference: Update at noon based on noon temp and afternoon forecast
The TD Breakthrough: Update estimates based on other estimates!
import numpy as np
import matplotlib.pyplot as plt
from collections import defaultdict, deque
import random
class TemporalDifferenceLearning:
"""Temporal Difference learning algorithms"""
def __init__(self, env, gamma=0.9, alpha=0.1):
self.env = env
self.gamma = gamma
self.alpha = alpha # Learning rate
def td_prediction(self, policy, num_episodes=1000):
"""
TD(0) prediction algorithm
V(s) ← V(s) + α[r + γV(s') - V(s)]
"""
# Initialize value function
V = defaultdict(float)
for episode in range(num_episodes):
state = self.env.reset()
done = False
while not done:
# Get action from policy
if isinstance(policy, dict):
action = policy.get(state, 0)
else:
action = policy(state)
# Take action
next_state, reward, done = self.env.step(action)
# TD update
td_target = reward + self.gamma * V[next_state] * (not done)
td_error = td_target - V[state]
V[state] += self.alpha * td_error
state = next_state
if (episode + 1) % 100 == 0:
print(f"Episode {episode + 1}/{num_episodes}")
return V
def sarsa(self, num_episodes=5000, epsilon=0.1):
"""
SARSA algorithm: On-policy TD control
Q(s,a) ← Q(s,a) + α[r + γQ(s',a') - Q(s,a)]
"""
# Initialize Q-function
Q = defaultdict(lambda: np.zeros(self.env.action_space.n))
# Create epsilon-greedy policy
def epsilon_greedy_policy(state, epsilon):
if np.random.random() < epsilon:
return np.random.randint(self.env.action_space.n)
else:
return np.argmax(Q[state])
# Track performance
episode_rewards = []
episode_lengths = []
for episode in range(num_episodes):
state = self.env.reset()
action = epsilon_greedy_policy(state, epsilon)
total_reward = 0
steps = 0
done = False
while not done:
# Take action
next_state, reward, done = self.env.step(action)
total_reward += reward
steps += 1
# Choose next action
next_action = epsilon_greedy_policy(next_state, epsilon)
# SARSA update
td_target = reward + self.gamma * Q[next_state][next_action] * (not done)
td_error = td_target - Q[state][action]
Q[state][action] += self.alpha * td_error
# Update state and action
state = next_state
action = next_action
episode_rewards.append(total_reward)
episode_lengths.append(steps)
# Decay epsilon
epsilon = max(0.01, epsilon * 0.995)
if (episode + 1) % 500 == 0:
avg_reward = np.mean(episode_rewards[-500:])
print(f"Episode {episode + 1}: Avg reward = {avg_reward:.2f}")
# Extract deterministic policy
policy = {}
for state in Q:
policy[state] = np.argmax(Q[state])
return policy, Q, episode_rewards, episode_lengths
def q_learning(self, num_episodes=5000, epsilon=0.1):
"""
Q-learning algorithm: Off-policy TD control
Q(s,a) ← Q(s,a) + α[r + γ max_a' Q(s',a') - Q(s,a)]
"""
# Initialize Q-function
Q = defaultdict(lambda: np.zeros(self.env.action_space.n))
# Track performance
episode_rewards = []
episode_lengths = []
for episode in range(num_episodes):
state = self.env.reset()
total_reward = 0
steps = 0
done = False
while not done:
# Epsilon-greedy action selection
if np.random.random() < epsilon:
action = np.random.randint(self.env.action_space.n)
else:
action = np.argmax(Q[state])
# Take action
next_state, reward, done = self.env.step(action)
total_reward += reward
steps += 1
# Q-learning update
if done:
td_target = reward
else:
td_target = reward + self.gamma * np.max(Q[next_state])
td_error = td_target - Q[state][action]
Q[state][action] += self.alpha * td_error
# Update state
state = next_state
episode_rewards.append(total_reward)
episode_lengths.append(steps)
# Decay epsilon
epsilon = max(0.01, epsilon * 0.995)
if (episode + 1) % 500 == 0:
avg_reward = np.mean(episode_rewards[-500:])
print(f"Episode {episode + 1}: Avg reward = {avg_reward:.2f}")
# Extract deterministic policy
policy = {}
for state in Q:
policy[state] = np.argmax(Q[state])
return policy, Q, episode_rewards, episode_lengths
def expected_sarsa(self, num_episodes=5000, epsilon=0.1):
"""
Expected SARSA algorithm
Q(s,a) ← Q(s,a) + α[r + γ 𝔼[Q(s',a')] - Q(s,a)]
"""
# Initialize Q-function
Q = defaultdict(lambda: np.zeros(self.env.action_space.n))
# Track performance
episode_rewards = []
episode_lengths = []
for episode in range(num_episodes):
state = self.env.reset()
total_reward = 0
steps = 0
done = False
while not done:
# Epsilon-greedy action selection
if np.random.random() < epsilon:
action = np.random.randint(self.env.action_space.n)
else:
action = np.argmax(Q[state])
# Take action
next_state, reward, done = self.env.step(action)
total_reward += reward
steps += 1
# Expected SARSA update
if done:
expected_next_q = 0
else:
# Calculate expected value of next state
max_q = np.max(Q[next_state])
# Count number of actions with max Q-value
n_max_actions = np.sum(Q[next_state] == max_q)
# Calculate probabilities for epsilon-greedy policy
greedy_prob = 1 - epsilon + epsilon / self.env.action_space.n
non_greedy_prob = epsilon / self.env.action_space.n
# Calculate expected value
expected_next_q = 0
for a in range(self.env.action_space.n):
if Q[next_state][a] == max_q:
prob = greedy_prob / n_max_actions if n_max_actions > 0 else 0
else:
prob = non_greedy_prob
expected_next_q += prob * Q[next_state][a]
td_target = reward + self.gamma * expected_next_q
td_error = td_target - Q[state][action]
Q[state][action] += self.alpha * td_error
# Update state
state = next_state
episode_rewards.append(total_reward)
episode_lengths.append(steps)
# Decay epsilon
epsilon = max(0.01, epsilon * 0.995)
# Extract deterministic policy
policy = {}
for state in Q:
policy[state] = np.argmax(Q[state])
return policy, Q, episode_rewards, episode_lengthsclass EligibilityTraces:
"""TD(λ) with eligibility traces"""
def __init__(self, env, gamma=0.9, alpha=0.1, lambda_param=0.7):
self.env = env
self.gamma = gamma
self.alpha = alpha
self.lambda_param = lambda_param
def td_lambda_prediction(self, policy, num_episodes=1000):
"""
TD(λ) prediction with eligibility traces
"""
V = defaultdict(float) # State values
e = defaultdict(float) # Eligibility traces
episode_rewards = []
for episode in range(num_episodes):
# Reset eligibility traces
e.clear()
state = self.env.reset()
total_reward = 0
done = False
while not done:
# Get action from policy
if isinstance(policy, dict):
action = policy.get(state, 0)
else:
action = policy(state)
# Take action
next_state, reward, done = self.env.step(action)
total_reward += reward
# Calculate TD error
if done:
td_target = reward
else:
td_target = reward + self.gamma * V[next_state]
td_error = td_target - V[state]
# Update eligibility trace for current state
e[state] = e.get(state, 0) + 1
# Update all states
for s in list(e.keys()):
V[s] += self.alpha * td_error * e[s]
e[s] *= self.gamma * self.lambda_param
state = next_state
episode_rewards.append(total_reward)
return V, episode_rewards
def sarsa_lambda(self, num_episodes=2000, epsilon=0.1):
"""
SARSA(λ) algorithm with eligibility traces
"""
# Initialize Q-function and eligibility traces
Q = defaultdict(lambda: np.zeros(self.env.action_space.n))
E = defaultdict(lambda: np.zeros(self.env.action_space.n))
episode_rewards = []
episode_lengths = []
for episode in range(num_episodes):
# Reset eligibility traces
E.clear()
state = self.env.reset()
# Choose action
if np.random.random() < epsilon:
action = np.random.randint(self.env.action_space.n)
else:
action = np.argmax(Q[state])
total_reward = 0
steps = 0
done = False
while not done:
# Take action
next_state, reward, done = self.env.step(action)
total_reward += reward
steps += 1
# Choose next action
if np.random.random() < epsilon:
next_action = np.random.randint(self.env.action_space.n)
else:
next_action = np.argmax(Q[next_state])
# Calculate TD error
if done:
td_target = reward
else:
td_target = reward + self.gamma * Q[next_state][next_action]
td_error = td_target - Q[state][action]
# Update eligibility trace
E[state][action] = E[state][action] + 1
# Update all state-action pairs
for s in list(E.keys()):
for a in range(self.env.action_space.n):
Q[s][a] += self.alpha * td_error * E[s][a]
E[s][a] *= self.gamma * self.lambda_param
# Move to next state-action pair
state = next_state
action = next_action
episode_rewards.append(total_reward)
episode_lengths.append(steps)
# Decay epsilon
epsilon = max(0.01, epsilon * 0.995)
if (episode + 1) % 200 == 0:
avg_reward = np.mean(episode_rewards[-200:])
print(f"Episode {episode + 1}: Avg reward = {avg_reward:.2f}")
# Extract deterministic policy
policy = {}
for state in Q:
policy[state] = np.argmax(Q[state])
return policy, Q, episode_rewards, episode_lengthsdef compare_rl_algorithms():
"""Comprehensive comparison of RL algorithms"""
print("="*70)
print("COMPARING REINFORCEMENT LEARNING ALGORITHMS")
print("="*70)
# Create a simple grid world for testing
class ComparisonEnv:
def __init__(self, size=5):
self.size = size
self.goal = (size-1, size-1)
self.start = (0, 0)
self.current = self.start
# Actions: 0=up, 1=right, 2=down, 3=left
self.actions = [0, 1, 2, 3]
self.action_deltas = {
0: (-1, 0),
1: (0, 1),
2: (1, 0),
3: (0, -1)
}
def reset(self):
self.current = self.start
return self.current
def step(self, action):
r, c = self.current
dr, dc = self.action_deltas[action]
new_r = max(0, min(self.size-1, r + dr))
new_c = max(0, min(self.size-1, c + dc))
self.current = (new_r, new_c)
# Rewards
if self.current == self.goal:
reward = 10
done = True
else:
reward = -1 # Step penalty
done = False
return self.current, reward, done
@property
def action_space(self):
return type('ActionSpace', (), {'n': 4})()
env = ComparisonEnv(size=5)
print("\n1. TEST ENVIRONMENT:")
print(f" Grid size: {env.size}×{env.size}")
print(f" Start: {env.start}, Goal: {env.goal}")
print(f" Step penalty: -1, Goal reward: +10")
# Define a simple policy for prediction
def simple_policy(state):
"""Move toward goal: right if column < goal column, else down"""
r, c = state
goal_r, goal_c = env.goal
if c < goal_c:
return 1 # right
elif r < goal_r:
return 2 # down
else:
return 0 # up
print("\n2. ALGORITHM COMPARISON:")
print("\n A. Dynamic Programming (Value Iteration)")
print(" - Requires complete model (P and R)")
print(" - Updates all states simultaneously")
print(" - Guaranteed convergence to optimal")
print("\n B. Monte Carlo Methods")
print(" - Model-free: learns from episodes")
print(" - High variance, zero bias")
print(" - Must wait until end of episode")
print("\n C. Temporal Difference Learning")
print(" - Model-free: learns from transitions")
print(" - Low variance, some bias")
print(" - Updates after each step")
print(" - Bootstrapping: learns guess-from-guess")
print("\n3. IMPLEMENTING AND COMPARING:")
# Test TD prediction
print("\n Testing TD(0) Prediction...")
td_learner = TemporalDifferenceLearning(env, gamma=0.9, alpha=0.1)
V_td = td_learner.td_prediction(simple_policy, num_episodes=1000)
# Test Monte Carlo prediction for comparison
print("\n Testing Monte Carlo Prediction...")
class MonteCarloSimple:
def __init__(self, env, gamma=0.9):
self.env = env
self.gamma = gamma
self.V = defaultdict(float)
self.returns = defaultdict(list)
def mc_prediction(self, policy, num_episodes=1000):
for episode in range(num_episodes):
# Generate episode
episode_history = []
state = env.reset()
done = False
while not done:
action = policy(state)
next_state, reward, done = env.step(action)
episode_history.append((state, action, reward))
state = next_state
# Calculate returns
G = 0
visited = set()
for t in reversed(range(len(episode_history))):
state, action, reward = episode_history[t]
G = self.gamma * G + reward
if state not in visited:
visited.add(state)
self.returns[state].append(G)
self.V[state] = np.mean(self.returns[state])
return self.V
mc_learner = MonteCarloSimple(env, gamma=0.9)
V_mc = mc_learner.mc_prediction(simple_policy, num_episodes=1000)
# Compare results
print("\n4. COMPARISON RESULTS:")
print("\n Value estimates for start state (0,0):")
print(f" TD(0): {V_td.get((0,0), 0):.3f}")
print(f" Monte Carlo: {V_mc.get((0,0), 0):.3f}")
# Theoretical optimal value (for comparison)
print("\n Theoretical optimal (shortest path):")
print(" Shortest path: 8 steps (7 right + 1 down)")
print(" Reward: 7×(-1) + 10 = 3")
print(" Discounted: -1 -0.9 -0.81 -0.73 -0.66 -0.59 -0.53 + 10×0.48 = 1.68")
print("\n5. CONTROL ALGORITHMS COMPARISON:")
# Test different control algorithms
print("\n Testing SARSA (on-policy TD control)...")
policy_sarsa, Q_sarsa, rewards_sarsa, lengths_sarsa = td_learner.sarsa(
num_episodes=2000, epsilon=0.1
)
print("\n Testing Q-learning (off-policy TD control)...")
policy_q, Q_q, rewards_q, lengths_q = td_learner.q_learning(
num_episodes=2000, epsilon=0.1
)
print("\n Testing Expected SARSA...")
policy_expected, Q_expected, rewards_expected, lengths_expected = td_learner.expected_sarsa(
num_episodes=2000, epsilon=0.1
)
# Plot comparison
plt.figure(figsize=(15, 10))
# Learning curves
plt.subplot(2, 3, 1)
window = 50
sarsa_smooth = np.convolve(rewards_sarsa, np.ones(window)/window, mode='valid')
q_smooth = np.convolve(rewards_q, np.ones(window)/window, mode='valid')
expected_smooth = np.convolve(rewards_expected, np.ones(window)/window, mode='valid')
plt.plot(sarsa_smooth, label='SARSA', alpha=0.8)
plt.plot(q_smooth, label='Q-learning', alpha=0.8)
plt.plot(expected_smooth, label='Expected SARSA', alpha=0.8)
plt.xlabel('Episode')
plt.ylabel('Average Reward (50-episode window)')
plt.title('Learning Curves Comparison')
plt.legend()
plt.grid(True, alpha=0.3)
# Final performance
plt.subplot(2, 3, 2)
final_rewards = [
np.mean(rewards_sarsa[-100:]),
np.mean(rewards_q[-100:]),
np.mean(rewards_expected[-100:])
]
algorithms = ['SARSA', 'Q-learning', 'Expected SARSA']
colors = ['blue', 'red', 'green']
bars = plt.bar(algorithms, final_rewards, color=colors, alpha=0.7)
plt.ylabel('Average Reward (last 100 episodes)')
plt.title('Final Performance Comparison')
# Add value labels on bars
for bar, reward in zip(bars, final_rewards):
plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.1,
f'{reward:.2f}', ha='center', va='bottom')
# Episode lengths
plt.subplot(2, 3, 3)
final_lengths = [
np.mean(lengths_sarsa[-100:]),
np.mean(lengths_q[-100:]),
np.mean(lengths_expected[-100:])
]
bars = plt.bar(algorithms, final_lengths, color=colors, alpha=0.7)
plt.ylabel('Average Steps (last 100 episodes)')
plt.title('Efficiency Comparison')
# Add value labels
for bar, length in zip(bars, final_lengths):
plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.1,
f'{length:.1f}', ha='center', va='bottom')
# Q-value distributions
plt.subplot(2, 3, 4)
q_values = []
for Q_dict, label in [(Q_sarsa, 'SARSA'), (Q_q, 'Q-learning'), (Q_expected, 'Expected SARSA')]:
values = []
for state in Q_dict:
values.extend(Q_dict[state])
q_values.append(values)
plt.boxplot(q_values, labels=algorithms)
plt.ylabel('Q-value')
plt.title('Q-value Distribution')
plt.grid(True, alpha=0.3)
# Convergence speed
plt.subplot(2, 3, 5)
convergence_points = []
for rewards, label in [(rewards_sarsa, 'SARSA'), (rewards_q, 'Q-learning'), (rewards_expected, 'Expected SARSA')]:
target = final_rewards[algorithms.index(label)] * 0.9 # 90% of final performance
for i in range(len(rewards)):
if np.mean(rewards[max(0, i-49):i+1]) >= target:
convergence_points.append(i)
break
bars = plt.bar(algorithms, convergence_points, color=colors, alpha=0.7)
plt.ylabel('Episodes to 90% convergence')
plt.title('Convergence Speed')
for bar, point in zip(bars, convergence_points):
plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 10,
f'{point}', ha='center', va='bottom')
# Policy visualization (heatmap)
plt.subplot(2, 3, 6)
def create_policy_grid(policy):
grid = np.zeros((env.size, env.size))
for r in range(env.size):
for c in range(env.size):
state = (r, c)
if state in policy:
grid[r, c] = policy[state]
else:
grid[r, c] = -1
return grid
policy_grid = create_policy_grid(policy_q)
im = plt.imshow(policy_grid, cmap='viridis', interpolation='nearest')
plt.colorbar(im, label='Action (0-3)')
plt.title('Q-learning Policy (example)')
plt.xlabel('Column')
plt.ylabel('Row')
plt.tight_layout()
plt.show()
print("\n6. KEY DIFFERENCES:")
print("\n SARSA (on-policy):")
print(" - Learns Q-values for the policy being followed")
print(" - More conservative (accounts for exploration)")
print(" - Better for stochastic environments")
print("\n Q-learning (off-policy):")
print(" - Learns optimal Q-values directly")
print(" - More aggressive (always assumes optimal action next)")
print(" - Can be more efficient but may overestimate")
print("\n Expected SARSA:")
print(" - Takes expectation over next actions")
print(" - Reduces variance compared to SARSA")
print(" - Can be used on-policy or off-policy")
print("\n7. PRACTICAL RECOMMENDATIONS:")
print(" - Use SARSA when exploration affects next action")
print(" - Use Q-learning for deterministic environments")
print(" - Use Expected SARSA to reduce variance")
print(" - TD methods generally faster than Monte Carlo")
print(" - Eligibility traces (TD(λ)) can speed up learning")
return {
'SARSA': (policy_sarsa, rewards_sarsa, lengths_sarsa),
'Q-learning': (policy_q, rewards_q, lengths_q),
'Expected SARSA': (policy_expected, rewards_expected, lengths_expected)
}
# Run the comparison
comparison_results = compare_rl_algorithms()class NStepTD:
"""n-step TD methods"""
def __init__(self, env, gamma=0.9, alpha=0.1):
self.env = env
self.gamma = gamma
self.alpha = alpha
def n_step_td_prediction(self, policy, n=3, num_episodes=1000):
"""
n-step TD prediction
"""
V = defaultdict(float)
for episode in range(num_episodes):
# Initialize
states = [self.env.reset()]
rewards = [0]
T = float('inf')
t = 0
while True:
if t < T:
# Take action
state = states[t]
if isinstance(policy, dict):
action = policy.get(state, 0)
else:
action = policy(state)
next_state, reward, done = self.env.step(action)
# Store
states.append(next_state)
rewards.append(reward)
if done:
T = t + 1
# Update time of interest
tau = t - n + 1
if tau >= 0:
# Calculate n-step return
G = 0
for i in range(tau + 1, min(tau + n, T) + 1):
G += (self.gamma ** (i - tau - 1)) * rewards[i]
if tau + n < T:
G += (self.gamma ** n) * V[states[tau + n]]
# Update value
V[states[tau]] += self.alpha * (G - V[states[tau]])
if tau == T - 1:
break
t += 1
return V
def n_step_sarsa(self, n=3, num_episodes=2000, epsilon=0.1):
"""
n-step SARSA algorithm
"""
Q = defaultdict(lambda: np.zeros(self.env.action_space.n))
episode_rewards = []
for episode in range(num_episodes):
# Initialize
states = [self.env.reset()]
actions = []
rewards = [0]
# Choose initial action
state = states[0]
if np.random.random() < epsilon:
action = np.random.randint(self.env.action_space.n)
else:
action = np.argmax(Q[state])
actions.append(action)
T = float('inf')
t = 0
while True:
if t < T:
# Take action
next_state, reward, done = self.env.step(actions[t])
# Store
states.append(next_state)
rewards.append(reward)
if done:
T = t + 1
else:
# Choose next action
if np.random.random() < epsilon:
next_action = np.random.randint(self.env.action_space.n)
else:
next_action = np.argmax(Q[next_state])
actions.append(next_action)
# Update time
tau = t - n + 1
if tau >= 0:
# Calculate n-step return
G = 0
for i in range(tau + 1, min(tau + n, T) + 1):
G += (self.gamma ** (i - tau - 1)) * rewards[i]
if tau + n < T:
state_tau_n = states[tau + n]
action_tau_n = actions[tau + n]
G += (self.gamma ** n) * Q[state_tau_n][action_tau_n]
# Update Q-value
state_tau = states[tau]
action_tau = actions[tau]
Q[state_tau][action_tau] += self.alpha * (G - Q[state_tau][action_tau])
if tau == T - 1:
break
t += 1
# Calculate episode reward
episode_reward = sum(rewards[1:]) # rewards[0] is 0
episode_rewards.append(episode_reward)
# Decay epsilon
epsilon = max(0.01, epsilon * 0.995)
# Extract policy
policy = {}
for state in Q:
policy[state] = np.argmax(Q[state])
return policy, Q, episode_rewards
def tree_backup(self, n=3, num_episodes=2000, epsilon=0.1):
"""
Tree Backup algorithm (n-step off-policy without importance sampling)
"""
Q = defaultdict(lambda: np.zeros(self.env.action_space.n))
episode_rewards = []
for episode in range(num_episodes):
# Initialize
states = [self.env.reset()]
actions = []
rewards = [0]
# Choose initial action
state = states[0]
if np.random.random() < epsilon:
action = np.random.randint(self.env.action_space.n)
else:
action = np.argmax(Q[state])
actions.append(action)
T = float('inf')
t = 0
while True:
if t < T:
# Take action
next_state, reward, done = self.env.step(actions[t])
# Store
states.append(next_state)
rewards.append(reward)
if done:
T = t + 1
else:
# Choose next action
if np.random.random() < epsilon:
next_action = np.random.randint(self.env.action_space.n)
else:
next_action = np.argmax(Q[next_state])
actions.append(next_action)
# Update time
tau = t - n + 1
if tau >= 0:
if t + 1 >= T:
G = rewards[T]
else:
# Calculate expected value for next state
next_state = states[tau + 1]
next_action = actions[tau + 1]
# Probability of greedy action
max_q = np.max(Q[next_state])
n_max = np.sum(Q[next_state] == max_q)
greedy_prob = 1 - epsilon + epsilon / n_max if n_max > 0 else 0
non_greedy_prob = epsilon / self.env.action_space.n
# Calculate expected Q-value
expected_q = 0
for a in range(self.env.action_space.n):
if Q[next_state][a] == max_q:
prob = greedy_prob
else:
prob = non_greedy_prob
expected_q += prob * Q[next_state][a]
G = rewards[tau + 1] + self.gamma * expected_q
# Backup through tree
for k in range(min(t, T-1), tau, -1):
state_k = states[k]
action_k = actions[k]
# Calculate expected value for state_k
max_q = np.max(Q[state_k])
n_max = np.sum(Q[state_k] == max_q)
greedy_prob = 1 - epsilon + epsilon / n_max if n_max > 0 else 0
non_greedy_prob = epsilon / self.env.action_space.n
expected_q = 0
for a in range(self.env.action_space.n):
if a != action_k:
if Q[state_k][a] == max_q:
prob = greedy_prob
else:
prob = non_greedy_prob
expected_q += prob * Q[state_k][a]
G = rewards[k] + self.gamma * expected_q
# Update Q-value
state_tau = states[tau]
action_tau = actions[tau]
Q[state_tau][action_tau] += self.alpha * (G - Q[state_tau][action_tau])
if tau == T - 1:
break
t += 1
episode_rewards.append(sum(rewards[1:]))
epsilon = max(0.01, epsilon * 0.995)
policy = {}
for state in Q:
policy[state] = np.argmax(Q[state])
return policy, Q, episode_rewardsdef cliff_walking_project():
"""Cliff Walking environment for TD learning comparison"""
print("="*70)
print("CLIFF WALKING PROJECT")
print("="*70)
class CliffWalkingEnv:
"""Cliff Walking environment from Sutton & Barto"""
def __init__(self, width=12, height=4):
self.width = width
self.height = height
# Start position (bottom-left)
self.start = (height-1, 0)
# Goal position (bottom-right)
self.goal = (height-1, width-1)
# Cliff positions (all cells in bottom row except start and goal)
self.cliff = [(height-1, c) for c in range(1, width-1)]
# Current position
self.position = list(self.start)
# Actions: 0=up, 1=right, 2=down, 3=left
self.actions = [0, 1, 2, 3]
self.action_deltas = {
0: (-1, 0),
1: (0, 1),
2: (1, 0),
3: (0, -1)
}
# For compatibility with TD learning class
self.action_space = type('ActionSpace', (), {'n': 4})()
def reset(self):
"""Reset to start position"""
self.position = list(self.start)
return tuple(self.position)
def step(self, action):
"""Take action"""
r, c = self.position
dr, dc = self.action_deltas[action]
# Move
new_r = max(0, min(self.height-1, r + dr))
new_c = max(0, min(self.width-1, c + dc))
self.position = [new_r, new_c]
current_pos = tuple(self.position)
# Check for cliff
if current_pos in self.cliff:
# Fall off cliff
reward = -100
self.position = list(self.start) # Reset to start
done = False # Not terminal, just reset
elif current_pos == self.goal:
# Reached goal
reward = 0 # No penalty for reaching goal
done = True
else:
# Normal step
reward = -1 # Step penalty
done = False
return tuple(self.position), reward, done
def render(self):
"""Render current state"""
grid = [['.' for _ in range(self.width)] for _ in range(self.height)]
# Mark cliff
for r, c in self.cliff:
grid[r][c] = 'C' # Cliff
# Mark start and goal
grid[self.start[0]][self.start[1]] = 'S'
grid[self.goal[0]][self.goal[1]] = 'G'
# Mark agent
r, c = self.position
if (r, c) != self.start and (r, c) != self.goal and (r, c) not in self.cliff:
grid[r][c] = 'A'
# Print
print("Cliff Walking Environment:")
print("S = Start, G = Goal, C = Cliff, A = Agent, . = Safe")
print("+" + "-" * (2*self.width + 1) + "+")
for row in grid:
print("| " + " ".join(row) + " |")
print("+" + "-" * (2*self.width + 1) + "+")
print(f"Position: {self.position}")
print("\n1. CREATING CLIFF WALKING ENVIRONMENT...")
env = CliffWalkingEnv(width=12, height=4)
print("\n2. ENVIRONMENT LAYOUT:")
env.render()
print("\n3. THE DILEMMA:")
print(" Safe path: Go up, right across top, then down")
print(" Risky path: Walk along edge of cliff")
print(" Optimal depends on learning algorithm!")
print("\n4. TESTING DIFFERENT TD ALGORITHMS...")
# Create TD learner
td_learner = TemporalDifferenceLearning(env, gamma=1.0, alpha=0.1)
# Test SARSA (on-policy)
print("\n A. SARSA (on-policy)...")
policy_sarsa, Q_sarsa, rewards_sarsa, lengths_sarsa = td_learner.sarsa(
num_episodes=500, epsilon=0.1
)
# Test Q-learning (off-policy)
print("\n B. Q-learning (off-policy)...")
policy_q, Q_q, rewards_q, lengths_q = td_learner.q_learning(
num_episodes=500, epsilon=0.1
)
# Test Expected SARSA
print("\n C. Expected SARSA...")
policy_expected, Q_expected, rewards_expected, lengths_expected = td_learner.expected_sarsa(
num_episodes=500, epsilon=0.1
)
print("\n5. RESULTS ANALYSIS:")
# Calculate statistics
def calculate_path_stats(policy):
"""Calculate statistics for a given policy"""
env_test = CliffWalkingEnv(width=12, height=4)
num_tests = 100
rewards = []
cliff_falls = []
paths = []
for test in range(num_tests):
state = env_test.reset()
path = [state]
total_reward = 0
cliff_count = 0
done = False
while not done and len(path) < 100:
if state in policy:
action = policy[state]
else:
action = np.random.randint(4)
state, reward, done = env_test.step(action)
path.append(state)
total_reward += reward
if tuple(state) in env_test.cliff:
cliff_count += 1
rewards.append(total_reward)
cliff_falls.append(cliff_count)
paths.append(path)
return {
'avg_reward': np.mean(rewards),
'std_reward': np.std(rewards),
'avg_cliff_falls': np.mean(cliff_falls),
'avg_path_length': np.mean([len(p) for p in paths]),
'paths': paths
}
stats_sarsa = calculate_path_stats(policy_sarsa)
stats_q = calculate_path_stats(policy_q)
stats_expected = calculate_path_stats(policy_expected)
print("\n Performance Statistics (100 test episodes):")
print("\n Algorithm | Avg Reward | Cliff Falls | Path Length")
print(" " + "-"*48)
print(f" SARSA | {stats_sarsa['avg_reward']:10.2f} | "
f"{stats_sarsa['avg_cliff_falls']:11.2f} | {stats_sarsa['avg_path_length']:11.2f}")
print(f" Q-learning | {stats_q['avg_reward']:10.2f} | "
f"{stats_q['avg_cliff_falls']:11.2f} | {stats_q['avg_path_length']:11.2f}")
print(f" Expected SARSA| {stats_expected['avg_reward']:10.2f} | "
f"{stats_expected['avg_cliff_falls']:11.2f} | {stats_expected['avg_path_length']:11.2f}")
print("\n6. VISUALIZING LEARNED POLICIES...")
# Create policy visualization
def visualize_policy(policy, title):
"""Visualize policy on cliff walking grid"""
fig, ax = plt.subplots(figsize=(12, 4))
# Create grid
grid = np.zeros((env.height, env.width))
# Mark different areas
for r in range(env.height):
for c in range(env.width):
pos = (r, c)
if pos == env.start:
grid[r, c] = 0.8 # Start (light blue)
elif pos == env.goal:
grid[r, c] = 0.6 # Goal (green)
elif pos in env.cliff:
grid[r, c] = 0.2 # Cliff (red)
else:
grid[r, c] = 0.4 # Safe (blue)
# Show grid
im = ax.imshow(grid, cmap='tab20c', aspect='auto')
# Add policy arrows
action_symbols = {0: '↑', 1: '→', 2: '↓', 3: '←'}
for r in range(env.height):
for c in range(env.width):
pos = (r, c)
if pos in policy and pos not in env.cliff and pos != env.goal:
action = policy[pos]
symbol = action_symbols.get(action, '?')
# Color based on safety
if r == env.height - 1 and c > 0 and c < env.width - 1:
color = 'white' # Cliff edge
else:
color = 'black'
ax.text(c, r, symbol, ha='center', va='center',
fontsize=10, fontweight='bold', color=color)
ax.set_title(title)
ax.set_xlabel('Column')
ax.set_ylabel('Row')
ax.set_xticks(range(env.width))
ax.set_yticks(range(env.height))
# Add legend
from matplotlib.patches import Patch
legend_elements = [
Patch(facecolor='lightblue', edgecolor='black', label='Start'),
Patch(facecolor='lightgreen', edgecolor='black', label='Goal'),
Patch(facecolor='lightcoral', edgecolor='black', label='Cliff'),
Patch(facecolor='skyblue', edgecolor='black', label='Safe')
]
ax.legend(handles=legend_elements, loc='upper right')
return fig, ax
# Create comparison figure
fig, axes = plt.subplots(1, 3, figsize=(18, 6))
for idx, (policy, title) in enumerate([
(policy_sarsa, 'SARSA Policy (On-policy)'),
(policy_q, 'Q-learning Policy (Off-policy)'),
(policy_expected, 'Expected SARSA Policy')
]):
ax = axes[idx]
# Create grid
grid = np.zeros((env.height, env.width))
for r in range(env.height):
for c in range(env.width):
pos = (r, c)
if pos == env.start:
grid[r, c] = 0.8
elif pos == env.goal:
grid[r, c] = 0.6
elif pos in env.cliff:
grid[r, c] = 0.2
else:
grid[r, c] = 0.4
ax.imshow(grid, cmap='tab20c', aspect='auto')
# Add policy arrows
action_symbols = {0: '↑', 1: '→', 2: '↓', 3: '←'}
for r in range(env.height):
for c in range(env.width):
pos = (r, c)
if pos in policy and pos not in env.cliff and pos != env.goal:
action = policy[pos]
symbol = action_symbols.get(action, '?')
color = 'white' if r == env.height - 1 and c > 0 and c < env.width - 1 else 'black'
ax.text(c, r, symbol, ha='center', va='center',
fontsize=10, fontweight='bold', color=color)
ax.set_title(title)
ax.set_xlabel('Column')
ax.set_ylabel('Row')
ax.set_xticks(range(env.width))
ax.set_yticks(range(env.height))
plt.tight_layout()
plt.show()
print("\n7. LEARNING CURVES COMPARISON...")
# Plot learning curves
plt.figure(figsize=(15, 5))
# Raw rewards
plt.subplot(1, 3, 1)
plt.plot(rewards_sarsa, alpha=0.5, label='SARSA')
plt.plot(rewards_q, alpha=0.5, label='Q-learning')
plt.plot(rewards_expected, alpha=0.5, label='Expected SARSA')
plt.xlabel('Episode')
plt.ylabel('Reward')
plt.title('Raw Learning Curves')
plt.legend()
plt.grid(True, alpha=0.3)
# Smoothed rewards
plt.subplot(1, 3, 2)
window = 20
sarsa_smooth = np.convolve(rewards_sarsa, np.ones(window)/window, mode='valid')
q_smooth = np.convolve(rewards_q, np.ones(window)/window, mode='valid')
expected_smooth = np.convolve(rewards_expected, np.ones(window)/window, mode='valid')
plt.plot(sarsa_smooth, label='SARSA', linewidth=2)
plt.plot(q_smooth, label='Q-learning', linewidth=2)
plt.plot(expected_smooth, label='Expected SARSA', linewidth=2)
plt.xlabel('Episode')
plt.ylabel('Average Reward (20-episode window)')
plt.title('Smoothed Learning Curves')
plt.legend()
plt.grid(True, alpha=0.3)
# Path lengths
plt.subplot(1, 3, 3)
length_smooth_sarsa = np.convolve(lengths_sarsa, np.ones(window)/window, mode='valid')
length_smooth_q = np.convolve(lengths_q, np.ones(window)/window, mode='valid')
length_smooth_expected = np.convolve(lengths_expected, np.ones(window)/window, mode='valid')
plt.plot(length_smooth_sarsa, label='SARSA', linewidth=2)
plt.plot(length_smooth_q, label='Q-learning', linewidth=2)
plt.plot(length_smooth_expected, label='Expected SARSA', linewidth=2)
plt.xlabel('Episode')
plt.ylabel('Average Path Length')
plt.title('Convergence of Path Length')
plt.legend()
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\n8. KEY INSIGHTS FROM CLIFF WALKING:")
print("\n SARSA (On-policy):")
print(" - Learns safe path away from cliff")
print(" - Accounts for exploration (epsilon-greedy)")
print(" - More conservative but safer")
print("\n Q-learning (Off-policy):")
print(" - Learns optimal (shortest) path")
print(" - May walk along cliff edge")
print(" - Can fall off cliff during exploration")
print(" - More efficient but riskier")
print("\n Expected SARSA:")
print(" - Compromise between SARSA and Q-learning")
print(" - Takes expectation over possible next actions")
print(" - Often best of both worlds")
print("\n9. PRACTICAL IMPLICATIONS:")
print(" - Use on-policy methods (SARSA) for safety-critical applications")
print(" - Use off-policy methods (Q-learning) for maximum efficiency")
print(" - Consider Expected SARSA for balanced performance")
print(" - The choice depends on exploration strategy and risk tolerance")
return {
'env': env,
'SARSA': (policy_sarsa, rewards_sarsa, stats_sarsa),
'Q-learning': (policy_q, rewards_q, stats_q),
'Expected SARSA': (policy_expected, rewards_expected, stats_expected)
}
# Run Cliff Walking project
cliff_results = cliff_walking_project()Chapter 6 Key Takeaways:
-
TD Learning Combines MC and DP:
- Like MC: Model-free, learns from experience
- Like DP: Bootstraps, updates estimates from other estimates
-
TD(0) Update:
V(s) ← V(s) + α[r + γV(s') - V(s)]- α: Learning rate
- r + γV(s'): TD target
- r + γV(s') - V(s): TD error
-
SARSA vs Q-learning:
-
SARSA (on-policy):
Q(s,a) ← Q(s,a) + α[r + γQ(s',a') - Q(s,a)] -
Q-learning (off-policy):
Q(s,a) ← Q(s,a) + α[r + γ max_a' Q(s',a') - Q(s,a)] - SARSA learns the policy being followed
- Q-learning learns the optimal policy directly
-
SARSA (on-policy):
-
Expected SARSA:
- Takes expectation over next actions
- Reduces variance compared to SARSA
- Can be on-policy or off-policy
-
TD(λ) and Eligibility Traces:
- λ = 0: Pure TD(0)
- λ = 1: Monte Carlo
- 0 < λ < 1: Intermediate, combines MC and TD
- Eligibility traces credit recent states
-
n-step Methods:
- n = 1: TD(0)
- n = ∞: Monte Carlo
- Intermediate n: Tradeoff between bias and variance
-
Practical Considerations:
- On-policy methods safer for exploration
- Off-policy methods more efficient
- Expected SARSA often provides best balance
- TD methods converge faster than MC in practice
The Curse of Dimensionality: Imagine a chess board with 64 squares. Now imagine:
- Each square can have 6 different pieces × 2 colors = 12 possibilities
- Total states: 12⁶⁴ ≈ 10⁶⁹
- More states than atoms in the observable universe!
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import SGDRegressor
from collections import defaultdict
class FunctionApproximationDemo:
"""Demonstrating why we need function approximation"""
def __init__(self):
self.initialize_examples()
def initialize_examples(self):
"""Initialize examples showing curse of dimensionality"""
print("="*70)
print("THE CURSE OF DIMENSIONALITY")
print("="*70)
# Example 1: State space explosion
print("\n1. STATE SPACE EXPLOSION:")
print("\n Consider a robot with n sensors, each with k values:")
for n in [2, 4, 8, 16]:
states = 10 ** n # 10 possible values per sensor
print(f" {n:2d} sensors: {states:,} states")
if states > 1e9:
print(f" (More than 1 billion states!)")
# Example 2: Continuous spaces
print("\n2. CONTINUOUS STATE SPACES:")
print("\n Real-world states are often continuous:")
print(" - Robot position: (x, y, θ) ∈ ℝ³")
print(" - Camera image: 640×480×3 ≈ 1 million continuous values")
print(" - Tabular methods impossible!")
# Example 3: Memory requirements
print("\n3. MEMORY REQUIREMENTS:")
print("\n Tabular Q-learning for Atari games:")
print(" - State: 210×160×3 = 100,800 pixels")
print(" - Each pixel: 256 values")
print(" - Total states: 256¹⁰⁰⁸⁰⁰ ≈ 10²⁴³⁰⁰⁰")
print(" - Known universe: ~10⁸⁰ atoms")
print(" - Need smarter representation!")
# Visual demonstration
self.visualize_curse()
def visualize_curse(self):
"""Visualize the curse of dimensionality"""
# Grid sizes for different dimensions
dimensions = [1, 2, 3, 4, 5]
grid_points_per_dim = 10
total_points = []
for d in dimensions:
points = grid_points_per_dim ** d
total_points.append(points)
plt.figure(figsize=(10, 6))
# Plot 1: Total points vs dimensions
plt.subplot(1, 2, 1)
plt.plot(dimensions, total_points, 'bo-', linewidth=2, markersize=8)
plt.yscale('log')
plt.xlabel('Number of Dimensions')
plt.ylabel('Total Grid Points (log scale)')
plt.title('State Space Explosion')
plt.grid(True, alpha=0.3)
# Annotate
for d, points in zip(dimensions, total_points):
plt.text(d, points, f'{points:,}', ha='center', va='bottom')
# Plot 2: Coverage fraction
plt.subplot(1, 2, 2)
# Fraction of space covered by fixed number of samples
samples = 10000
coverage_fractions = []
for d in dimensions:
# Volume of d-dimensional hypercube
total_volume = 1.0
# Fraction covered by samples randomly distributed
# This is simplified: fraction ≈ samples / (grid_points_per_dim**d)
fraction = min(1.0, samples / (grid_points_per_dim ** d))
coverage_fractions.append(fraction)
plt.plot(dimensions, coverage_fractions, 'ro-', linewidth=2, markersize=8)
plt.xlabel('Number of Dimensions')
plt.ylabel('Fraction of Space Covered')
plt.title(f'Coverage with {samples:,} Samples')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
# Run the demo
demo = FunctionApproximationDemo()Core Idea: Instead of storing Q(s,a) in a table, approximate it with a function:
Q(s,a) ≈ f_θ(s,a)
where θ are the function parameters.
class LinearFunctionApproximation:
"""Linear function approximation for value functions"""
def __init__(self, feature_dim, num_actions, learning_rate=0.01):
"""
Initialize linear function approximator
Args:
feature_dim: Dimension of feature vector
num_actions: Number of possible actions
learning_rate: Learning rate for gradient descent
"""
self.feature_dim = feature_dim
self.num_actions = num_actions
self.learning_rate = learning_rate
# Initialize weights: one weight vector per action
# θ[a] ∈ ℝ^{feature_dim}
self.theta = np.random.randn(num_actions, feature_dim) * 0.01
# For tracking
self.loss_history = []
def extract_features(self, state):
"""
Extract feature vector from state
To be implemented by subclasses for specific problems
"""
# Default: one-hot encoding for discrete states
if isinstance(state, int):
features = np.zeros(self.feature_dim)
if state < self.feature_dim:
features[state] = 1
return features
else:
# Assume state is already a feature vector
return np.array(state).flatten()
def predict(self, state):
"""
Predict Q-values for all actions
Returns:
Q_values: array of length num_actions
"""
features = self.extract_features(state)
# Linear prediction: Q(s,a) = θ[a]·φ(s)
Q_values = self.theta @ features
return Q_values
def update(self, state, action, target):
"""
Update weights using gradient descent
Args:
state: Current state
action: Action taken
target: Target Q-value
"""
features = self.extract_features(state)
# Current prediction
prediction = self.theta[action] @ features
# Compute error
error = target - prediction
# Gradient descent update
# ∇_θ J(θ) = - (target - θ·φ(s)) φ(s)
# θ ← θ + α (target - θ·φ(s)) φ(s)
gradient = features * error
self.theta[action] += self.learning_rate * gradient
# Track loss
self.loss_history.append(error ** 2)
return error ** 2 # Return squared error
def sarsa_update(self, state, action, reward, next_state, next_action,
gamma=0.9, done=False):
"""
SARSA update with function approximation
"""
features = self.extract_features(state)
next_features = self.extract_features(next_state)
# Current Q-value
current_q = self.theta[action] @ features
# Next Q-value
if done:
target = reward
else:
next_q = self.theta[next_action] @ next_features
target = reward + gamma * next_q
# Update
error = target - current_q
gradient = features * error
self.theta[action] += self.learning_rate * gradient
self.loss_history.append(error ** 2)
return error ** 2
def q_learning_update(self, state, action, reward, next_state,
gamma=0.9, done=False):
"""
Q-learning update with function approximation
"""
features = self.extract_features(state)
next_features = self.extract_features(next_state)
# Current Q-value
current_q = self.theta[action] @ features
# Max Q-value for next state
if done:
target = reward
else:
next_q_values = self.theta @ next_features
max_next_q = np.max(next_q_values)
target = reward + gamma * max_next_q
# Update
error = target - current_q
gradient = features * error
self.theta[action] += self.learning_rate * gradient
self.loss_history.append(error ** 2)
return error ** 2
def get_policy(self, state, epsilon=0.0):
"""
Get epsilon-greedy policy for a state
"""
if np.random.random() < epsilon:
return np.random.randint(self.num_actions)
else:
Q_values = self.predict(state)
return np.argmax(Q_values)
def visualize_weights(self):
"""Visualize the learned weights"""
plt.figure(figsize=(12, 4))
plt.subplot(1, 3, 1)
plt.imshow(self.theta, aspect='auto', cmap='RdYlBu')
plt.colorbar(label='Weight value')
plt.xlabel('Feature dimension')
plt.ylabel('Action')
plt.title('Weight Matrix θ')
plt.subplot(1, 3, 2)
weight_norms = np.linalg.norm(self.theta, axis=1)
plt.bar(range(self.num_actions), weight_norms)
plt.xlabel('Action')
plt.ylabel('Weight norm')
plt.title('Weight Norms per Action')
plt.subplot(1, 3, 3)
if self.loss_history:
window = min(100, len(self.loss_history) // 10)
if window > 0:
smoothed_loss = np.convolve(self.loss_history,
np.ones(window)/window,
mode='valid')
plt.plot(smoothed_loss)
plt.xlabel('Update step')
plt.ylabel('Smoothed squared error')
plt.title('Learning Progress')
plt.yscale('log')
plt.tight_layout()
plt.show()class FeatureEngineering:
"""Different feature representations for RL"""
@staticmethod
def one_hot_encoding(state, n_states):
"""One-hot encoding for discrete states"""
features = np.zeros(n_states)
if isinstance(state, (int, np.integer)):
if state < n_states:
features[state] = 1
return features
@staticmethod
def tile_coding(state, n_tilings=8, n_tiles=8, offset=0.0):
"""
Tile coding (CMAC) for continuous states
Args:
state: Continuous state value in [0, 1]
n_tilings: Number of tilings
n_tiles: Number of tiles per tiling
offset: Offset for each tiling
Returns:
Sparse feature vector
"""
if isinstance(state, (int, np.integer)):
# Discrete state: convert to continuous
state = state / 100.0
# Ensure state is in [0, 1]
state = max(0.0, min(1.0, state))
# Create features
features = np.zeros(n_tilings * n_tiles)
for tiling in range(n_tilings):
# Add offset to each tiling
offset_state = state + tiling * offset
# Find which tile this state belongs to
tile_idx = int(offset_state * n_tiles) % n_tiles
# Set feature
feature_idx = tiling * n_tiles + tile_idx
if feature_idx < len(features):
features[feature_idx] = 1
return features
@staticmethod
def polynomial_features(state, degree=2):
"""
Polynomial features for continuous states
Args:
state: Can be scalar or vector
degree: Maximum polynomial degree
Returns:
Polynomial feature vector
"""
state = np.array(state).flatten()
n_features = len(state)
# Generate all polynomial combinations up to given degree
from itertools import combinations_with_replacement
features = []
# Bias term (degree 0)
features.append(1.0)
# Terms of increasing degree
for d in range(1, degree + 1):
# All combinations of features with total degree d
for comb in combinations_with_replacement(range(n_features), d):
term = 1.0
for idx in comb:
term *= state[idx]
features.append(term)
return np.array(features)
@staticmethod
def radial_basis_functions(state, centers, sigma=1.0):
"""
Radial basis function features
Args:
state: Input state (scalar or vector)
centers: Array of RBF centers
sigma: Width parameter
Returns:
RBF feature vector
"""
state = np.array(state).flatten()
centers = np.array(centers)
if state.ndim == 1:
state = state.reshape(1, -1)
# Compute distances to all centers
if centers.ndim == 1:
centers = centers.reshape(-1, 1)
# For scalar states
if len(state.shape) == 1 or state.shape[1] == 1:
distances = np.abs(state - centers.T)
else:
# For vector states
distances = np.linalg.norm(state - centers, axis=1)
# Apply Gaussian kernel
features = np.exp(-distances**2 / (2 * sigma**2))
return features.flatten()
@staticmethod
def fourier_features(state, order=3):
"""
Fourier basis features
Args:
state: Normalized to [0, 1]
order: Maximum frequency
Returns:
Fourier feature vector
"""
if isinstance(state, (int, np.integer)):
state = state / 100.0
state = max(0.0, min(1.0, state))
features = []
# Constant term
features.append(1.0)
# Sin and cos terms
for k in range(1, order + 1):
features.append(np.sin(2 * np.pi * k * state))
features.append(np.cos(2 * np.pi * k * state))
return np.array(features)
def compare_feature_representations():
"""Compare different feature representations"""
print("="*70)
print("FEATURE REPRESENTATION COMPARISON")
print("="*70)
# Generate example states
n_samples = 100
states = np.linspace(0, 1, n_samples)
# Create feature representations
features_dict = {
'Tile Coding (8x8)': [],
'Polynomial (degree 3)': [],
'RBF (10 centers)': [],
'Fourier (order 3)': []
}
fe = FeatureEngineering()
# Tile coding
for state in states:
features = fe.tile_coding(state, n_tilings=8, n_tiles=8, offset=1/8)
features_dict['Tile Coding (8x8)'].append(features)
# Polynomial features
for state in states:
features = fe.polynomial_features([state], degree=3)
features_dict['Polynomial (degree 3)'].append(features)
# RBF features
centers = np.linspace(0, 1, 10)
for state in states:
features = fe.radial_basis_functions(state, centers, sigma=0.2)
features_dict['RBF (10 centers)'].append(features)
# Fourier features
for state in states:
features = fe.fourier_features(state, order=3)
features_dict['Fourier (order 3)'].append(features)
# Visualize
fig, axes = plt.subplots(2, 2, figsize=(15, 10))
axes = axes.flatten()
for idx, (title, features_list) in enumerate(features_dict.items()):
ax = axes[idx]
features_array = np.array(features_list)
# Show feature matrix
im = ax.imshow(features_array.T, aspect='auto', cmap='viridis',
extent=[0, 1, 0, features_array.shape[1]])
ax.set_xlabel('State (normalized)')
ax.set_ylabel('Feature index')
ax.set_title(f'{title}\nShape: {features_array.shape}')
# Add colorbar
plt.colorbar(im, ax=ax)
plt.tight_layout()
plt.show()
print("\nFEATURE REPRESENTATION PROPERTIES:")
print("\n1. Tile Coding (CMAC):")
print(" - Good for coarse coding")
print(" - Generalization through overlapping tiles")
print(" - Efficient with sparse representation")
print("\n2. Polynomial Features:")
print(" - Can approximate smooth functions")
print(" - Captures interactions between features")
print(" - Can suffer from curse of dimensionality")
print("\n3. Radial Basis Functions:")
print(" - Localized representation")
print(" - Good for interpolation")
print(" - Center placement is crucial")
print("\n4. Fourier Basis:")
print(" - Good for periodic functions")
print(" - Captures frequency information")
print(" - Can approximate any continuous function")
print("\n5. Neural Networks (next chapter):")
print(" - Learned features (no manual engineering)")
print(" - Can handle high-dimensional inputs")
print(" - Requires more data and computation")
return features_dict
# Run feature comparison
feature_comparison = compare_feature_representations()class LinearTDAgent:
"""TD learning with linear function approximation"""
def __init__(self, env, feature_dim, num_actions,
learning_rate=0.01, gamma=0.9, epsilon=0.1):
self.env = env
self.feature_dim = feature_dim
self.num_actions = num_actions
self.learning_rate = learning_rate
self.gamma = gamma
self.epsilon = epsilon
# Initialize function approximator
self.approximator = LinearFunctionApproximation(
feature_dim, num_actions, learning_rate
)
# For tracking
self.episode_rewards = []
self.episode_lengths = []
def extract_features(self, state):
"""Extract features from state - to be overridden"""
# Default implementation for discrete states
return FeatureEngineering.one_hot_encoding(state, self.feature_dim)
def train_sarsa(self, num_episodes=1000):
"""Train using SARSA with linear function approximation"""
for episode in range(num_episodes):
state = self.env.reset()
features = self.extract_features(state)
# Choose action
action = self.approximator.get_policy(features, self.epsilon)
total_reward = 0
steps = 0
done = False
while not done:
# Take action
next_state, reward, done = self.env.step(action)
next_features = self.extract_features(next_state)
# Choose next action
next_action = self.approximator.get_policy(next_features, self.epsilon)
# SARSA update
self.approximator.sarsa_update(
features, action, reward, next_features, next_action,
self.gamma, done
)
# Update tracking
total_reward += reward
steps += 1
# Move to next state
state = next_state
features = next_features
action = next_action
self.episode_rewards.append(total_reward)
self.episode_lengths.append(steps)
# Decay epsilon
self.epsilon = max(0.01, self.epsilon * 0.995)
if (episode + 1) % 100 == 0:
avg_reward = np.mean(self.episode_rewards[-100:])
print(f"Episode {episode + 1}: Avg reward = {avg_reward:.2f}")
return self.episode_rewards, self.episode_lengths
def train_q_learning(self, num_episodes=1000):
"""Train using Q-learning with linear function approximation"""
for episode in range(num_episodes):
state = self.env.reset()
features = self.extract_features(state)
total_reward = 0
steps = 0
done = False
while not done:
# Choose action
action = self.approximator.get_policy(features, self.epsilon)
# Take action
next_state, reward, done = self.env.step(action)
next_features = self.extract_features(next_state)
# Q-learning update
self.approximator.q_learning_update(
features, action, reward, next_features,
self.gamma, done
)
# Update tracking
total_reward += reward
steps += 1
# Move to next state
state = next_state
features = next_features
self.episode_rewards.append(total_reward)
self.episode_lengths.append(steps)
# Decay epsilon
self.epsilon = max(0.01, self.epsilon * 0.995)
if (episode + 1) % 100 == 0:
avg_reward = np.mean(self.episode_rewards[-100:])
print(f"Episode {episode + 1}: Avg reward = {avg_reward:.2f}")
return self.episode_rewards, self.episode_lengths
def evaluate(self, num_episodes=100, render=False):
"""Evaluate the learned policy"""
total_rewards = []
for episode in range(num_episodes):
state = self.env.reset()
features = self.extract_features(state)
total_reward = 0
done = False
if render:
self.env.render()
while not done:
# Greedy action (no exploration)
action = self.approximator.get_policy(features, epsilon=0.0)
# Take action
state, reward, done = self.env.step(action)
features = self.extract_features(state)
total_reward += reward
if render:
self.env.render()
total_rewards.append(total_reward)
avg_reward = np.mean(total_rewards)
std_reward = np.std(total_rewards)
print(f"Evaluation over {num_episodes} episodes:")
print(f" Average reward: {avg_reward:.2f} ± {std_reward:.2f}")
print(f" Best episode: {np.max(total_rewards):.2f}")
print(f" Worst episode: {np.min(total_rewards):.2f}")
return total_rewards
def visualize_learning(self):
"""Visualize learning progress"""
plt.figure(figsize=(15, 5))
# Plot rewards
plt.subplot(1, 3, 1)
window = min(100, len(self.episode_rewards) // 10)
if window > 0:
smoothed_rewards = np.convolve(self.episode_rewards,
np.ones(window)/window,
mode='valid')
plt.plot(smoothed_rewards)
plt.xlabel('Episode')
plt.ylabel(f'Average Reward ({window}-episode window)')
plt.title('Learning Curve')
plt.grid(True, alpha=0.3)
# Plot episode lengths
plt.subplot(1, 3, 2)
if window > 0:
smoothed_lengths = np.convolve(self.episode_lengths,
np.ones(window)/window,
mode='valid')
plt.plot(smoothed_lengths)
plt.xlabel('Episode')
plt.ylabel(f'Average Length ({window}-episode window)')
plt.title('Episode Lengths')
plt.grid(True, alpha=0.3)
# Plot value function
plt.subplot(1, 3, 3)
if hasattr(self.env, 'get_all_states'):
states = self.env.get_all_states()
values = []
for state in states:
features = self.extract_features(state)
Q_values = self.approximator.predict(features)
values.append(np.max(Q_values))
plt.plot(states, values, 'b-', linewidth=2)
plt.xlabel('State')
plt.ylabel('Max Q-value')
plt.title('Learned Value Function')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()def mountain_car_project():
"""Mountain Car with function approximation"""
print("="*70)
print("MOUNTAIN CAR PROJECT")
print("="*70)
class MountainCarEnv:
"""Simplified Mountain Car environment"""
def __init__(self):
# State: [position, velocity]
self.min_position = -1.2
self.max_position = 0.6
self.max_speed = 0.07
self.goal_position = 0.5
# Actions: 0=left, 1=neutral, 2=right
self.actions = [0, 1, 2]
self.action_space = type('ActionSpace', (), {'n': 3})()
self.reset()
def reset(self):
"""Reset environment"""
# Start near bottom of hill
self.position = -0.5
self.velocity = 0.0
return self._get_state()
def step(self, action):
"""Take action"""
# Update velocity
if action == 0: # left
force = -1.0
elif action == 2: # right
force = 1.0
else: # neutral
force = 0.0
# Physics update
self.velocity += force * 0.001 - 0.0025 * np.cos(3 * self.position)
self.velocity = np.clip(self.velocity, -self.max_speed, self.max_speed)
self.position += self.velocity
self.position = np.clip(self.position, self.min_position, self.max_position)
# Reset velocity if hit left boundary
if self.position == self.min_position and self.velocity < 0:
self.velocity = 0.0
# Check if done
done = (self.position >= self.goal_position)
# Reward
if done:
reward = 0.0 # Success!
else:
reward = -1.0 # Step penalty
return self._get_state(), reward, done
def _get_state(self):
"""Get current state"""
return np.array([self.position, self.velocity])
def render(self):
"""Simple text render"""
print(f"Position: {self.position:.3f}, Velocity: {self.velocity:.3f}")
def get_state_bounds(self):
"""Get bounds for state normalization"""
return {
'position': (self.min_position, self.max_position),
'velocity': (-self.max_speed, self.max_speed)
}
print("\n1. UNDERSTANDING MOUNTAIN CAR:")
print("\n The Challenge:")
print(" - Car starts at bottom of valley")
print(" - Engine not powerful enough to drive straight up")
print(" - Must build momentum by swinging back and forth")
print(" - State: continuous (position, velocity)")
print(" - Tabular methods impossible!")
env = MountainCarEnv()
print("\n2. CREATING FEATURE REPRESENTATION:")
class MountainCarAgent(LinearTDAgent):
"""Mountain Car agent with tile coding"""
def __init__(self, env, n_tilings=8, n_tiles=8):
# Calculate feature dimension
feature_dim = n_tilings * n_tiles * n_tiles # 2D tile coding
super().__init__(env, feature_dim, env.action_space.n,
learning_rate=0.1 / n_tilings, # Common heuristic
gamma=1.0, # Episodic task
epsilon=0.1)
self.n_tilings = n_tilings
self.n_tiles = n_tiles
# Get state bounds for normalization
bounds = env.get_state_bounds()
self.pos_bounds = bounds['position']
self.vel_bounds = bounds['velocity']
# Create tiling offsets
self.offsets = []
for i in range(n_tilings):
# Different offset for each tiling
pos_offset = (i * 1.0) / n_tilings
vel_offset = (i * 1.7) / n_tilings # Irrational for better coverage
self.offsets.append((pos_offset, vel_offset))
def extract_features(self, state):
"""2D tile coding for Mountain Car"""
position, velocity = state
# Normalize to [0, 1]
pos_norm = (position - self.pos_bounds[0]) / (self.pos_bounds[1] - self.pos_bounds[0])
vel_norm = (velocity - self.vel_bounds[0]) / (self.vel_bounds[1] - self.vel_bounds[0])
# Initialize feature vector
features = np.zeros(self.feature_dim)
for tiling in range(self.n_tilings):
# Add offsets
pos_offset, vel_offset = self.offsets[tiling]
pos_off = pos_norm + pos_offset
vel_off = vel_norm + vel_offset
# Find tile indices
pos_idx = int(pos_off * self.n_tiles) % self.n_tiles
vel_idx = int(vel_off * self.n_tiles) % self.n_tiles
# Calculate feature index
feature_idx = (tiling * self.n_tiles * self.n_tiles +
pos_idx * self.n_tiles + vel_idx)
if feature_idx < self.feature_dim:
features[feature_idx] = 1.0
return features
def visualize_value_function(self):
"""Visualize learned value function"""
# Create grid of states
pos_values = np.linspace(self.pos_bounds[0], self.pos_bounds[1], 50)
vel_values = np.linspace(self.vel_bounds[0], self.vel_bounds[1], 50)
values = np.zeros((len(pos_values), len(vel_values)))
for i, pos in enumerate(pos_values):
for j, vel in enumerate(vel_values):
state = np.array([pos, vel])
features = self.extract_features(state)
Q_values = self.approximator.predict(features)
values[i, j] = np.max(Q_values)
# Plot
plt.figure(figsize=(12, 10))
# 3D surface plot
from mpl_toolkits.mplot3d import Axes3D
X, Y = np.meshgrid(vel_values, pos_values)
ax = plt.subplot(1, 2, 1, projection='3d')
surf = ax.plot_surface(X, Y, values, cmap='viridis',
linewidth=0, antialiased=True)
ax.set_xlabel('Velocity')
ax.set_ylabel('Position')
ax.set_zlabel('Value')
ax.set_title('3D Value Function')
plt.colorbar(surf, ax=ax, shrink=0.5)
# 2D heatmap
ax = plt.subplot(1, 2, 2)
im = ax.imshow(values, extent=[vel_values[0], vel_values[-1],
pos_values[0], pos_values[-1]],
aspect='auto', cmap='viridis', origin='lower')
ax.set_xlabel('Velocity')
ax.set_ylabel('Position')
ax.set_title('Value Function Heatmap')
# Add goal line
ax.axhline(y=env.goal_position, color='red', linestyle='--',
alpha=0.5, label='Goal position')
# Add sample trajectories
ax.legend()
plt.colorbar(im, ax=ax)
plt.tight_layout()
plt.show()
def visualize_policy(self):
"""Visualize learned policy"""
# Create grid of states
pos_values = np.linspace(self.pos_bounds[0], self.pos_bounds[1], 30)
vel_values = np.linspace(self.vel_bounds[0], self.vel_bounds[1], 30)
actions_grid = np.zeros((len(pos_values), len(vel_values)))
for i, pos in enumerate(pos_values):
for j, vel in enumerate(vel_values):
state = np.array([pos, vel])
features = self.extract_features(state)
action = self.approximator.get_policy(features, epsilon=0.0)
actions_grid[i, j] = action
# Plot
plt.figure(figsize=(10, 8))
# Policy heatmap
plt.imshow(actions_grid, extent=[vel_values[0], vel_values[-1],
pos_values[0], pos_values[-1]],
aspect='auto', cmap='coolwarm', origin='lower')
plt.xlabel('Velocity')
plt.ylabel('Position')
plt.title('Learned Policy (0=left, 1=neutral, 2=right)')
# Add contour lines for value function
from scipy import ndimage
# Smooth the policy for contours
smoothed = ndimage.gaussian_filter(actions_grid, sigma=1.0)
contour_levels = np.arange(-0.5, 3.5, 0.5)
plt.contour(vel_values, pos_values, smoothed, levels=contour_levels,
colors='black', alpha=0.3, linewidths=0.5)
# Add goal line
plt.axhline(y=env.goal_position, color='red', linestyle='--',
alpha=0.7, label='Goal position')
plt.colorbar(label='Action')
plt.legend()
plt.tight_layout()
plt.show()
print("\n3. TRAINING AGENT WITH TILE CODING...")
print(f" Feature dimension: 8 tilings × 8×8 tiles = 512 features")
print(" Learning rate: α = 0.1/8 = 0.0125 (common heuristic)")
agent = MountainCarAgent(env, n_tilings=8, n_tiles=8)
print("\n4. RUNNING Q-LEARNING...")
rewards, lengths = agent.train_q_learning(num_episodes=2000)
print("\n5. VISUALIZING RESULTS...")
# Plot learning curve
plt.figure(figsize=(15, 5))
plt.subplot(1, 3, 1)
window = 50
smoothed_rewards = np.convolve(rewards, np.ones(window)/window, mode='valid')
plt.plot(smoothed_rewards)
plt.xlabel('Episode')
plt.ylabel(f'Average Reward ({window}-episode window)')
plt.title('Learning Curve')
plt.grid(True, alpha=0.3)
# Add success threshold (reaching goal gives 0 reward, else -1)
# Success means reward close to 0
plt.axhline(y=-50, color='red', linestyle='--', alpha=0.5,
label='Typical success threshold')
plt.legend()
plt.subplot(1, 3, 2)
# Histogram of final performance
final_rewards = rewards[-200:] # Last 200 episodes
plt.hist(final_rewards, bins=20, alpha=0.7, color='blue', edgecolor='black')
plt.xlabel('Episode Reward')
plt.ylabel('Frequency')
plt.title('Final Performance Distribution')
# Mark successful episodes (reward > -100)
success_rate = np.sum(np.array(final_rewards) > -100) / len(final_rewards)
plt.axvline(x=-100, color='red', linestyle='--', alpha=0.7,
label=f'Success rate: {success_rate:.1%}')
plt.legend()
plt.subplot(1, 3, 3)
# Episode lengths
smoothed_lengths = np.convolve(lengths, np.ones(window)/window, mode='valid')
plt.plot(smoothed_lengths)
plt.xlabel('Episode')
plt.ylabel(f'Average Length ({window}-episode window)')
plt.title('Episode Lengths')
plt.grid(True, alpha=0.3)
# Optimal length (around 110-130 steps for good policy)
plt.axhline(y=120, color='green', linestyle='--', alpha=0.5,
label='Good policy target')
plt.legend()
plt.tight_layout()
plt.show()
print("\n6. EVALUATING LEARNED POLICY...")
eval_rewards = agent.evaluate(num_episodes=100)
success_count = sum(1 for r in eval_rewards if r > -200) # Didn't timeout
print(f"\n Success rate: {success_count}/{len(eval_rewards)} ({success_count/len(eval_rewards):.1%})")
print(f" Average steps: {np.mean([abs(r) for r in eval_rewards]):.1f}")
print("\n7. VISUALIZING LEARNED VALUE FUNCTION AND POLICY...")
agent.visualize_value_function()
agent.visualize_policy()
print("\n8. DEMONSTRATING SUCCESSFUL EPISODE...")
# Run one episode with visualization
print("\n Starting demonstration episode...")
state = env.reset()
total_reward = 0
steps = 0
done = False
positions = []
velocities = []
actions_taken = []
while not done and steps < 200:
features = agent.extract_features(state)
action = agent.approximator.get_policy(features, epsilon=0.0)
state, reward, done = env.step(action)
positions.append(state[0])
velocities.append(state[1])
actions_taken.append(action)
total_reward += reward
steps += 1
print(f" Total reward: {total_reward}")
print(f" Steps taken: {steps}")
print(f" Reached goal: {state[0] >= env.goal_position}")
# Plot trajectory
plt.figure(figsize=(12, 4))
plt.subplot(1, 3, 1)
plt.plot(positions)
plt.xlabel('Step')
plt.ylabel('Position')
plt.title('Position Over Time')
plt.axhline(y=env.goal_position, color='red', linestyle='--', alpha=0.5)
plt.grid(True, alpha=0.3)
plt.subplot(1, 3, 2)
plt.plot(velocities)
plt.xlabel('Step')
plt.ylabel('Velocity')
plt.title('Velocity Over Time')
plt.grid(True, alpha=0.3)
plt.subplot(1, 3, 3)
# Phase plot
plt.plot(positions, velocities, 'b-', alpha=0.7)
plt.scatter(positions[0], velocities[0], color='green', s=100,
label='Start', zorder=5)
plt.scatter(positions[-1], velocities[-1], color='red', s=100,
label='End', zorder=5)
plt.xlabel('Position')
plt.ylabel('Velocity')
plt.title('Phase Space Trajectory')
plt.axvline(x=env.goal_position, color='red', linestyle='--', alpha=0.3)
plt.legend()
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\n9. KEY INSIGHTS FROM MOUNTAIN CAR:")
print("\n Why function approximation works:")
print(" 1. Tile coding generalizes similar states")
print(" 2. Learns to swing back and forth")
print(" 3. Builds momentum strategy")
print("\n Challenges faced:")
print(" 1. Sparse rewards (only at goal)")
print(" 2. Need to learn complex strategy")
print(" 3. Continuous state space")
print("\n Solutions demonstrated:")
print(" 1. Tile coding for continuous states")
print(" 2. Step penalty encourages efficiency")
print(" 3. Q-learning for off-policy learning")
return env, agent, rewards
# Run Mountain Car project
mountain_car_env, mountain_car_agent, mountain_car_rewards = mountain_car_project()def convergence_analysis():
"""Analyze convergence issues in function approximation"""
print("="*70)
print("CONVERGENCE ANALYSIS IN FUNCTION APPROXIMATION")
print("="*70)
# Create a simple environment for analysis
class ConvergenceEnv:
def __init__(self):
self.states = list(range(10))
self.actions = [0, 1]
self.action_space = type('ActionSpace', (), {'n': 2})()
# Define true optimal values
self.true_values = {s: np.sin(s/10 * 2*np.pi) for s in self.states}
def reset(self):
return np.random.choice(self.states)
def step(self, action):
state = np.random.choice(self.states) # Random transitions
reward = self.true_values[state] + np.random.normal(0, 0.1)
done = np.random.random() < 0.1 # 10% chance of episode ending
return state, reward, done
env = ConvergenceEnv()
print("\n1. THE CHALLENGE: NON-STATIONARY TARGETS")
print("\n In tabular TD:")
print(" V(s) ← V(s) + α[r + γV(s') - V(s)]")
print(" Target: r + γV(s')")
print(" V(s') fixed during update")
print("\n With function approximation:")
print(" θ ← θ + α[r + γV_θ(s') - V_θ(s)]∇V_θ(s)")
print(" Problem: V_θ(s') changes as θ changes!")
print(" Target depends on parameters being learned")
print(" Can cause divergence!")
print("\n2. DIVERGENCE EXAMPLE:")
class DivergenceDemo:
def __init__(self):
# Simple linear function: y = θx
self.theta = 1.0
self.x = 2.0
self.true_value = 1.0
def update(self, alpha=0.1, gamma=0.9):
"""TD update that can diverge"""
# Current prediction
prediction = self.theta * self.x
# Target (depends on current θ!)
target = self.true_value + gamma * prediction
# Update
error = target - prediction
gradient = self.x
self.theta += alpha * error * gradient
return prediction, target, self.theta
def demonstrate_divergence(self):
"""Show how TD can diverge with function approximation"""
print("\n Starting θ = 1.0, x = 2.0, true value = 1.0")
print("\n Iteration | Prediction | Target | New θ")
print(" " + "-"*40)
self.theta = 1.0
for i in range(10):
pred, target, new_theta = self.update(alpha=0.1, gamma=0.9)
print(f" {i+1:9d} | {pred:10.3f} | {target:6.3f} | {new_theta:6.3f}")
if abs(new_theta) > 10: # Diverged
print(f"\n DIVERGED after {i+1} iterations!")
break
demo = DivergenceDemo()
demo.demonstrate_divergence()
print("\n3. SOLUTIONS FOR CONVERGENCE:")
print("\n A. Gradient Temporal Difference (GTD):")
print(" - Uses two timescale updates")
print(" - Maintains auxiliary weights")
print(" - Provably convergent")
print("\n B. Target Networks:")
print(" - Use separate network for targets")
print(" - Target network updated slowly")
print(" - Stabilizes learning (used in DQN)")
print("\n C. Experience Replay:")
print(" - Store transitions in memory buffer")
print(" - Sample random mini-batches")
print(" - Breaks temporal correlations")
print("\n D. Proper Step Size Selection:")
print(" - Smaller learning rates")
print(" - Adaptive learning rates (Adam, RMSProp)")
print(" - Learning rate schedules")
print("\n4. IMPLEMENTING GTD:")
class GTDLearner:
"""Gradient Temporal Difference learning"""
def __init__(self, feature_dim, num_actions, alpha=0.01, beta=0.01):
self.feature_dim = feature_dim
self.num_actions = num_actions
# Primary weights
self.theta = np.zeros((num_actions, feature_dim))
# Auxiliary weights (for GTD)
self.w = np.zeros((num_actions, feature_dim))
self.alpha = alpha # Learning rate for θ
self.beta = beta # Learning rate for w
def extract_features(self, state):
"""Simple one-hot encoding"""
features = np.zeros(self.feature_dim)
if isinstance(state, int) and state < self.feature_dim:
features[state] = 1
return features
def predict(self, state):
features = self.extract_features(state)
return self.theta @ features
def gtd_update(self, state, action, reward, next_state,
next_action, gamma=0.9, done=False):
"""GTD update rule"""
features = self.extract_features(state)
next_features = self.extract_features(next_state)
# Current prediction
delta = reward - self.theta[action] @ features
if not done:
delta += gamma * (self.theta[next_action] @ next_features)
# GTD updates
# Update auxiliary weights w
self.w[action] += self.beta * (delta - self.w[action] @ features) * features
# Update primary weights θ
self.theta[action] += self.alpha * (
(features - gamma * next_features) * (self.w[action] @ features)
)
return delta ** 2
print("\n GTD Update Equations:")
print(" δ = r + γθ·φ(s') - θ·φ(s)")
print(" w ← w + β(δ - w·φ(s))φ(s)")
print(" θ ← θ + α(φ(s) - γφ(s'))(w·φ(s))")
print("\n5. COMPARING CONVERGENCE:")
# Create test environment
class TestConvergenceEnv:
def __init__(self, n_states=20):
self.n_states = n_states
self.states = list(range(n_states))
self.actions = [0, 1]
self.action_space = type('ActionSpace', (), {'n': 2})()
# Define true optimal Q-values
self.true_Q = {}
for s in self.states:
self.true_Q[s] = np.array([
np.sin(s/n_states * 2*np.pi),
np.cos(s/n_states * 2*np.pi)
])
def reset(self):
return np.random.choice(self.states)
def step(self, action):
# Simple deterministic MDP
state = self.reset()
# Reward = true Q + noise
reward = self.true_Q[state][action] + np.random.normal(0, 0.1)
# 10% chance episode ends
done = np.random.random() < 0.1
return state, reward, done
test_env = TestConvergenceEnv(n_states=20)
# Test different algorithms
algorithms = {
'TD': LinearFunctionApproximation(20, 2, learning_rate=0.05),
'GTD': GTDLearner(20, 2, alpha=0.05, beta=0.01)
}
# Training loop
num_episodes = 1000
results = {name: [] for name in algorithms}
for episode in range(num_episodes):
state = test_env.reset()
done = False
while not done:
# Random action for simplicity
action = np.random.choice([0, 1])
next_state, reward, done = test_env.step(action)
next_action = np.random.choice([0, 1])
# Update each algorithm
for name, algorithm in algorithms.items():
if name == 'TD':
error = algorithm.sarsa_update(
state, action, reward, next_state, next_action,
gamma=0.9, done=done
)
elif name == 'GTD':
error = algorithm.gtd_update(
state, action, reward, next_state, next_action,
gamma=0.9, done=done
)
results[name].append(error)
state = next_state
# Plot results
plt.figure(figsize=(15, 5))
# Plot 1: Error over time
plt.subplot(1, 3, 1)
for name, errors in results.items():
window = 100
smoothed = np.convolve(errors, np.ones(window)/window, mode='valid')
plt.plot(smoothed[:500], label=name, linewidth=2)
plt.xlabel('Update step')
plt.ylabel('Squared TD error (smoothed)')
plt.title('Convergence Comparison')
plt.legend()
plt.grid(True, alpha=0.3)
plt.yscale('log')
# Plot 2: Final error distribution
plt.subplot(1, 3, 2)
final_errors = {name: errors[-1000:] for name, errors in results.items()}
positions = np.arange(len(algorithms))
means = [np.mean(errors) for errors in final_errors.values()]
stds = [np.std(errors) for errors in final_errors.values()]
plt.bar(positions, means, yerr=stds, capsize=10, alpha=0.7)
plt.xticks(positions, list(algorithms.keys()))
plt.ylabel('Mean squared error (last 1000 updates)')
plt.title('Final Performance')
plt.grid(True, alpha=0.3)
# Plot 3: Parameter norm over time
plt.subplot(1, 3, 3)
# For demonstration, track parameter norms
theta_norms = []
for i in range(0, len(results['TD']), 100):
if i < len(results['TD']):
theta_norms.append(np.linalg.norm(algorithms['TD'].theta))
plt.plot(theta_norms, label='TD parameter norm', linewidth=2)
plt.xlabel('Update step (every 100th)')
plt.ylabel('Parameter norm ||θ||')
plt.title('Parameter Stability')
plt.legend()
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\n6. PRACTICAL ADVICE:")
print("\n When function approximation diverges:")
print(" 1. Reduce learning rate")
print(" 2. Use target networks")
print(" 3. Implement experience replay")
print(" 4. Try GTD for guaranteed convergence")
print(" 5. Normalize features")
print("\n Feature engineering tips:")
print(" 1. Tile coding works well for low-dimensional spaces")
print(" 2. Neural networks for high-dimensional spaces")
print(" 3. Normalize inputs to [-1, 1] or [0, 1]")
print(" 4. Consider feature interactions")
return algorithms, results
# Run convergence analysis
convergence_algorithms, convergence_results = convergence_analysis()Chapter 7 Key Takeaways:
-
The Curse of Dimensionality:
- Tabular methods fail for large/continuous state spaces
- Need compact representations
-
Function Approximation:
- Learn Q(s,a) ≈ f_θ(s,a) instead of table
- Parameters θ learned from data
- Generalizes to unseen states
-
Feature Representations:
- Tile Coding: Good for low-dimensional, overlapping features
- Polynomial Features: Captures interactions, can overfit
- Radial Basis Functions: Localized, good for interpolation
- Fourier Basis: Periodic functions, universal approximation
- Neural Networks: Learned features (next chapter)
-
Linear Function Approximation:
- Q(s,a) = θ[a]·φ(s)
- Simple, converges under right conditions
- Need good feature engineering
-
Convergence Issues:
- Non-stationary targets can cause divergence
- Solutions: GTD, target networks, experience replay
- Proper step sizes crucial
-
Mountain Car Example:
- Continuous state space needs function approximation
- Tile coding works well for 2D state
- Learns complex swinging strategy
-
Practical Considerations:
- Start with linear methods + good features
- Monitor for divergence
- Normalize features
- Use experience replay for stability
The 2013 Revolution: DeepMind's DQN could play Atari games at human level from pixels alone!
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import numpy as np
import matplotlib.pyplot as plt
from collections import deque, namedtuple
import random
import cv2
class DQNBreakthrough:
"""Understanding the DQN breakthrough"""
def __init__(self):
self.history = self.create_timeline()
def create_timeline(self):
"""Create timeline of DQN development"""
print("="*70)
print("THE DQN BREAKTHROUGH TIMELINE")
print("="*70)
timeline = [
("2013", "DeepMind publishes first DQN paper",
"Playing Atari with Deep Reinforcement Learning"),
("2015", "Nature paper: Human-level control",
"First AI to learn directly from pixels"),
("2016", "Improvements: Double DQN, Dueling DQN",
"Addressing overestimation bias"),
("2017", "Rainbow DQN",
"Combining 7 improvements for state-of-the-art"),
("2018", "Distributional DQN",
"Learning value distributions instead of expectations"),
("2020+", "Modern extensions",
"Prioritized replay, noisy nets, multi-step learning")
]
for year, title, description in timeline:
print(f"\n{year}: {title}")
print(f" {description}")
return timeline
def atari_challenge(self):
"""Explain the Atari learning challenge"""
print("\n" + "="*70)
print("THE ATARI LEARNING CHALLENGE")
print("="*70)
challenges = [
("Input", "210×160 RGB pixels (100,800 values)",
"Need to process raw pixels"),
("Action Space", "Up to 18 discrete actions",
"Learn which buttons to press"),
("State Space", "~256¹⁰⁰⁸⁰⁰ possible states",
"More than atoms in universe"),
("Rewards", "Sparse and delayed",
"Score only increases with achievements"),
("Generalization", "Learn from pixels alone",
"No game-specific knowledge")
]
print("\nChallenges faced by DQN:")
for challenge, numbers, description in challenges:
print(f"\n{challenge:15s}: {numbers}")
print(f"{'':15s} {description}")
def dqn_architecture_overview(self):
"""Show DQN architecture"""
print("\n" + "="*70)
print("DQN ARCHITECTURE OVERVIEW")
print("="*70)
architecture = """
Raw Pixels (210×160×3)
↓
Preprocessing (84×84×4 grayscale)
↓
Convolutional Layers (feature extraction)
↓
Fully Connected Layers
↓
Q-values for each action
Key Innovations:
1. Experience Replay: Break correlations
2. Target Network: Stabilize learning
3. Frame Stacking: Handle partial observability
"""
print(architecture)
# Visualize the architecture
self.visualize_dqn_architecture()
# Initialize
dqn_intro = DQNBreakthrough()
dqn_intro.atari_challenge()
dqn_intro.dqn_architecture_overview()class DQNNetwork(nn.Module):
"""Deep Q-Network architecture"""
def __init__(self, input_shape, n_actions):
"""
Initialize DQN network
Args:
input_shape: (channels, height, width)
n_actions: Number of possible actions
"""
super(DQNNetwork, self).__init__()
# Input shape
self.input_channels = input_shape[0]
self.input_height = input_shape[1]
self.input_width = input_shape[2]
# Convolutional layers (feature extraction)
self.conv1 = nn.Conv2d(self.input_channels, 32, kernel_size=8, stride=4)
self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)
self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)
# Calculate flattened size
conv_output_size = self._get_conv_output(input_shape)
# Fully connected layers
self.fc1 = nn.Linear(conv_output_size, 512)
self.fc2 = nn.Linear(512, n_actions)
# Initialize weights
self._initialize_weights()
def _get_conv_output(self, shape):
"""Calculate output size of convolutional layers"""
batch_size = 1
input = torch.autograd.Variable(torch.rand(batch_size, *shape))
output = self._forward_conv(input)
return output.view(batch_size, -1).size(1)
def _forward_conv(self, x):
"""Forward pass through convolutional layers"""
x = F.relu(self.conv1(x))
x = F.relu(self.conv2(x))
x = F.relu(self.conv3(x))
return x
def _initialize_weights(self):
"""Initialize network weights"""
for module in self.modules():
if isinstance(module, nn.Conv2d) or isinstance(module, nn.Linear):
nn.init.kaiming_normal_(module.weight, mode='fan_out', nonlinearity='relu')
if module.bias is not None:
nn.init.constant_(module.bias, 0)
def forward(self, x):
"""Forward pass"""
# Convolutional layers
x = self._forward_conv(x)
# Flatten
x = x.view(x.size(0), -1)
# Fully connected layers
x = F.relu(self.fc1(x))
x = self.fc2(x)
return x
def get_action(self, state, epsilon=0.0):
"""
Get action using epsilon-greedy policy
Args:
state: Preprocessed state tensor
epsilon: Exploration rate
Returns:
action: Integer action index
"""
if np.random.random() < epsilon:
# Explore: random action
return np.random.randint(self.fc2.out_features)
else:
# Exploit: best action
with torch.no_grad():
q_values = self.forward(state)
return torch.argmax(q_values).item()
class SimpleDQN(nn.Module):
"""Simplified DQN for non-visual inputs"""
def __init__(self, input_dim, hidden_dim, n_actions):
"""
Simple feedforward DQN
Args:
input_dim: Dimension of input state
hidden_dim: Size of hidden layers
n_actions: Number of possible actions
"""
super(SimpleDQN, self).__init__()
self.network = nn.Sequential(
nn.Linear(input_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, n_actions)
)
# Initialize weights
self._initialize_weights()
def _initialize_weights(self):
for layer in self.network:
if isinstance(layer, nn.Linear):
nn.init.kaiming_normal_(layer.weight, mode='fan_out', nonlinearity='relu')
nn.init.constant_(layer.bias, 0)
def forward(self, x):
return self.network(x)
def visualize_neural_network():
"""Visualize how neural networks approximate Q-functions"""
print("\n" + "="*70)
print("NEURAL NETWORKS AS FUNCTION APPROXIMATORS")
print("="*70)
# Create a simple 1D Q-function to approximate
def true_q_function(state, action):
"""True Q-function we want to learn"""
x = state[0]
if action == 0: # Move left
return -x**2
else: # Move right
return np.sin(2 * np.pi * x)
# Generate training data
n_samples = 1000
states = np.random.uniform(-1, 1, (n_samples, 1))
actions = np.random.randint(0, 2, n_samples)
targets = np.array([true_q_function([s], a) for s, a in zip(states, actions)])
# Create neural network
class QNetwork1D(nn.Module):
def __init__(self):
super(QNetwork1D, self).__init__()
self.net = nn.Sequential(
nn.Linear(2, 64), # state (1) + action (1) = 2
nn.ReLU(),
nn.Linear(64, 64),
nn.ReLU(),
nn.Linear(64, 1)
)
def forward(self, state, action):
# One-hot encode action
action_onehot = F.one_hot(action.long(), num_classes=2).float()
x = torch.cat([state, action_onehot], dim=1)
return self.net(x)
# Train the network
net = QNetwork1D()
optimizer = optim.Adam(net.parameters(), lr=0.001)
criterion = nn.MSELoss()
# Convert to tensors
states_tensor = torch.FloatTensor(states)
actions_tensor = torch.LongTensor(actions)
targets_tensor = torch.FloatTensor(targets).unsqueeze(1)
# Training loop
losses = []
for epoch in range(1000):
optimizer.zero_grad()
predictions = net(states_tensor, actions_tensor)
loss = criterion(predictions, targets_tensor)
loss.backward()
optimizer.step()
losses.append(loss.item())
# Test the learned function
test_states = np.linspace(-1, 1, 100).reshape(-1, 1)
test_states_tensor = torch.FloatTensor(test_states)
with torch.no_grad():
q_values_action0 = []
q_values_action1 = []
for state in test_states:
# Action 0
state_tensor = torch.FloatTensor(state).unsqueeze(0)
action0_tensor = torch.LongTensor([0])
q0 = net(state_tensor, action0_tensor).item()
q_values_action0.append(q0)
# Action 1
action1_tensor = torch.LongTensor([1])
q1 = net(state_tensor, action1_tensor).item()
q_values_action1.append(q1)
# Plot results
fig, axes = plt.subplots(2, 2, figsize=(15, 10))
# Plot 1: True vs learned Q-function for action 0
ax = axes[0, 0]
true_q0 = [-x**2 for x in test_states.flatten()]
ax.plot(test_states, true_q0, 'b-', label='True Q(s,0)', linewidth=3, alpha=0.7)
ax.plot(test_states, q_values_action0, 'r--', label='Learned Q(s,0)', linewidth=2)
ax.set_xlabel('State')
ax.set_ylabel('Q-value')
ax.set_title('Action 0: Move Left')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 2: True vs learned Q-function for action 1
ax = axes[0, 1]
true_q1 = [np.sin(2 * np.pi * x) for x in test_states.flatten()]
ax.plot(test_states, true_q1, 'b-', label='True Q(s,1)', linewidth=3, alpha=0.7)
ax.plot(test_states, q_values_action1, 'r--', label='Learned Q(s,1)', linewidth=2)
ax.set_xlabel('State')
ax.set_ylabel('Q-value')
ax.set_title('Action 1: Move Right')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 3: Optimal policy
ax = axes[1, 0]
optimal_actions = []
for q0, q1 in zip(q_values_action0, q_values_action1):
if q0 > q1:
optimal_actions.append(0)
else:
optimal_actions.append(1)
ax.plot(test_states, optimal_actions, 'g-', linewidth=2)
ax.set_xlabel('State')
ax.set_ylabel('Optimal Action')
ax.set_title('Learned Optimal Policy')
ax.set_yticks([0, 1])
ax.set_yticklabels(['Left', 'Right'])
ax.grid(True, alpha=0.3)
# Plot 4: Training loss
ax = axes[1, 1]
ax.plot(losses)
ax.set_xlabel('Training Step')
ax.set_ylabel('MSE Loss')
ax.set_title('Training Convergence')
ax.set_yscale('log')
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\nKEY INSIGHTS:")
print("1. Neural networks can approximate complex Q-functions")
print("2. Single network outputs Q-values for all actions")
print("3. Network learns smooth approximations")
print("4. Optimal policy emerges from Q-value comparisons")
# Visualize neural network approximation
visualize_neural_network()class ReplayBuffer:
"""Experience replay buffer for DQN"""
def __init__(self, capacity, device='cpu'):
"""
Initialize replay buffer
Args:
capacity: Maximum number of transitions to store
device: Device to store tensors on
"""
self.capacity = capacity
self.device = device
self.buffer = deque(maxlen=capacity)
# Define transition structure
self.Transition = namedtuple('Transition',
('state', 'action', 'reward',
'next_state', 'done'))
def push(self, state, action, reward, next_state, done):
"""Add a transition to the buffer"""
transition = self.Transition(state, action, reward, next_state, done)
self.buffer.append(transition)
def sample(self, batch_size):
"""
Sample a batch of transitions
Returns:
tuple of (states, actions, rewards, next_states, dones)
"""
if len(self.buffer) < batch_size:
raise ValueError(f"Buffer only has {len(self.buffer)} samples, need {batch_size}")
transitions = random.sample(self.buffer, batch_size)
batch = self.Transition(*zip(*transitions))
# Convert to tensors
state_batch = torch.stack(batch.state).to(self.device)
action_batch = torch.tensor(batch.action, device=self.device).unsqueeze(1)
reward_batch = torch.tensor(batch.reward, device=self.device, dtype=torch.float32).unsqueeze(1)
next_state_batch = torch.stack(batch.next_state).to(self.device)
done_batch = torch.tensor(batch.done, device=self.device, dtype=torch.float32).unsqueeze(1)
return (state_batch, action_batch, reward_batch,
next_state_batch, done_batch)
def __len__(self):
return len(self.buffer)
class PrioritizedReplayBuffer(ReplayBuffer):
"""Prioritized experience replay buffer"""
def __init__(self, capacity, alpha=0.6, beta=0.4, beta_increment=0.001, device='cpu'):
"""
Initialize prioritized replay buffer
Args:
capacity: Maximum buffer size
alpha: Priority exponent (0 = uniform, 1 = fully prioritized)
beta: Importance sampling exponent
beta_increment: Beta increment per sample
device: Device for tensors
"""
super().__init__(capacity, device)
self.alpha = alpha
self.beta = beta
self.beta_increment = beta_increment
self.priorities = np.zeros(capacity, dtype=np.float32)
self.position = 0
self.max_priority = 1.0
def push(self, state, action, reward, next_state, done):
"""Add transition with maximum priority"""
# Find position
if len(self.buffer) < self.capacity:
self.buffer.append(None)
# Store transition
transition = self.Transition(state, action, reward, next_state, done)
self.buffer[self.position] = transition
# Set priority
self.priorities[self.position] = self.max_priority ** self.alpha
# Update position
self.position = (self.position + 1) % self.capacity
def sample(self, batch_size):
"""Sample batch with priorities"""
if len(self.buffer) < batch_size:
raise ValueError(f"Buffer only has {len(self.buffer)} samples, need {batch_size}")
# Calculate sampling probabilities
priorities = self.priorities[:len(self.buffer)]
probabilities = priorities / priorities.sum()
# Sample indices
indices = np.random.choice(len(self.buffer), batch_size, p=probabilities)
# Get transitions
transitions = [self.buffer[idx] for idx in indices]
batch = self.Transition(*zip(*transitions))
# Convert to tensors
state_batch = torch.stack(batch.state).to(self.device)
action_batch = torch.tensor(batch.action, device=self.device).unsqueeze(1)
reward_batch = torch.tensor(batch.reward, device=self.device, dtype=torch.float32).unsqueeze(1)
next_state_batch = torch.stack(batch.next_state).to(self.device)
done_batch = torch.tensor(batch.done, device=self.device, dtype=torch.float32).unsqueeze(1)
# Calculate importance sampling weights
weights = (len(self.buffer) * probabilities[indices]) ** (-self.beta)
weights = weights / weights.max() # Normalize
weight_batch = torch.tensor(weights, device=self.device, dtype=torch.float32).unsqueeze(1)
# Update beta
self.beta = min(1.0, self.beta + self.beta_increment)
return (state_batch, action_batch, reward_batch,
next_state_batch, done_batch, indices, weight_batch)
def update_priorities(self, indices, priorities):
"""Update priorities for sampled transitions"""
for idx, priority in zip(indices, priorities):
self.priorities[idx] = (priority + 1e-5) ** self.alpha
self.max_priority = max(self.max_priority, priority)
def experience_replay_demo():
"""Demonstrate the power of experience replay"""
print("\n" + "="*70)
print("EXPERIENCE REPLAY: WHY IT MATTERS")
print("="*70)
# Create a simple environment
class CorrelatedEnv:
"""Environment that produces highly correlated samples"""
def __init__(self):
self.state = 0
self.states = list(range(100))
def reset(self):
self.state = 0
return self.state
def step(self, action):
# Highly correlated: always move to next state
self.state = (self.state + 1) % 100
reward = np.sin(self.state / 10)
done = False
return self.state, reward, done
env = CorrelatedEnv()
print("\nTHE PROBLEM: CORRELATED SAMPLES")
print("Sequential experiences are highly correlated:")
print("- Breaks i.i.d. assumption of most learning algorithms")
print("- Can cause unstable learning")
print("- Network forgets old experiences")
print("\n" + "-"*70)
print("SOLUTION: EXPERIENCE REPLAY")
print("-"*70)
print("\nHow it works:")
print("1. Store experiences (s, a, r, s', done) in replay buffer")
print("2. Sample random mini-batches from buffer")
print("3. Breaks temporal correlations")
print("4. Reuses experiences (data efficiency)")
# Demonstrate correlation
print("\nDEMONSTRATION:")
# Generate sequential experiences
sequential_states = []
state = env.reset()
for _ in range(20):
state, reward, done = env.step(0)
sequential_states.append(state)
print(f"\nSequential states (highly correlated):")
print(sequential_states)
# Create replay buffer
buffer = ReplayBuffer(capacity=1000)
# Fill buffer
state = env.reset()
for _ in range(100):
action = np.random.randint(0, 2)
next_state, reward, done = env.step(action)
# Store as tensor
state_tensor = torch.FloatTensor([state])
next_state_tensor = torch.FloatTensor([next_state])
buffer.push(state_tensor, action, reward, next_state_tensor, done)
state = next_state
# Sample from buffer
batch_states = []
for _ in range(5):
states, _, _, _, _ = buffer.sample(5)
batch_states.extend(states.numpy().flatten().tolist())
print(f"\nRandom samples from replay buffer (decorrelated):")
print([int(s) for s in batch_states[:20]])
# Show statistics
print("\nSTATISTICS:")
print(f"Sequential correlation: {np.corrcoef(sequential_states[:-1], sequential_states[1:])[0,1]:.3f}")
# Calculate correlation in replay buffer samples
if len(batch_states) > 1:
replay_corr = np.corrcoef(batch_states[:-1], batch_states[1:])[0,1]
print(f"Replay buffer correlation: {replay_corr:.3f}")
# Visualize the difference
plt.figure(figsize=(12, 5))
# Plot sequential samples
plt.subplot(1, 2, 1)
plt.plot(sequential_states, 'b-o', alpha=0.7)
plt.xlabel('Step')
plt.ylabel('State')
plt.title('Sequential Samples (High Correlation)')
plt.grid(True, alpha=0.3)
# Plot replay buffer samples
plt.subplot(1, 2, 2)
plt.plot(batch_states, 'r-o', alpha=0.7)
plt.xlabel('Sample Index')
plt.ylabel('State')
plt.title('Replay Buffer Samples (Low Correlation)')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\nBENEFITS OF EXPERIENCE REPLAY:")
print("1. Breaks temporal correlations")
print("2. Increases data efficiency (reuse experiences)")
print("3. Smoothes training distribution")
print("4. Enables mini-batch learning")
print("\nVARIATIONS:")
print("1. Prioritized Replay: Sample important transitions more often")
print("2. Hindsight Experience Replay: Learn from failures")
print("3. Distributed Replay: Multiple parallel actors")
return buffer
# Run experience replay demo
replay_buffer = experience_replay_demo()class TargetNetworkManager:
"""Manages target network updates"""
def __init__(self, policy_net, target_net, update_method='hard', tau=0.001):
"""
Initialize target network manager
Args:
policy_net: The network being trained
target_net: The network used for targets
update_method: 'hard' or 'soft' updates
tau: Soft update parameter (0 < tau ≤ 1)
"""
self.policy_net = policy_net
self.target_net = target_net
self.update_method = update_method
self.tau = tau
# Initially synchronize target network
self.hard_update()
def hard_update(self):
"""Hard update: copy all weights"""
self.target_net.load_state_dict(self.policy_net.state_dict())
def soft_update(self):
"""Soft update: polyak averaging"""
for target_param, policy_param in zip(self.target_net.parameters(),
self.policy_net.parameters()):
target_param.data.copy_(
self.tau * policy_param.data + (1 - self.tau) * target_param.data
)
def update(self, step=None):
"""
Update target network based on method
Args:
step: Current training step (for scheduled updates)
"""
if self.update_method == 'hard':
# Hard update every N steps
if step is not None and step % 1000 == 0:
self.hard_update()
elif self.update_method == 'soft':
# Soft update every step
self.soft_update()
else:
raise ValueError(f"Unknown update method: {self.update_method}")
def target_network_analysis():
"""Analyze why target networks are needed"""
print("\n" + "="*70)
print("TARGET NETWORKS: STABILIZING LEARNING")
print("="*70)
print("\nTHE PROBLEM: MOVING TARGETS")
print("\nWithout target networks:")
print("Q(s,a) ← Q(s,a) + α[r + γ max_a' Q(s',a') - Q(s,a)]")
print(" ↑")
print(" This target changes every step!")
print("\nResult: Unstable learning, can diverge")
print("\n" + "-"*70)
print("SOLUTION: TARGET NETWORKS")
print("-"*70)
print("\nWith target networks:")
print("Q(s,a) ← Q(s,a) + α[r + γ max_a' Q_target(s',a') - Q(s,a)]")
print(" ↑")
print(" Target network changes slowly")
print("\nTwo update methods:")
print("1. Hard updates: Copy weights every N steps")
print("2. Soft updates: Polyak averaging: θ_target ← τθ + (1-τ)θ_target")
# Demonstrate the problem
print("\n" + "-"*70)
print("DEMONSTRATION")
print("-"*70)
# Create a simple Q-function learning task
class MovingTargetEnv:
def __init__(self):
self.true_q = 5.0 # True optimal Q-value
self.current_estimate = 0.0
def step(self):
"""TD update without target network"""
# Simulate noisy reward
reward = self.true_q + np.random.normal(0, 1.0)
# TD target using current estimate (problem!)
target = reward + 0.9 * self.current_estimate
# Update
error = target - self.current_estimate
self.current_estimate += 0.1 * error
return self.current_estimate, error
def step_with_target(self, target_estimate):
"""TD update with target network"""
reward = self.true_q + np.random.normal(0, 1.0)
target = reward + 0.9 * target_estimate
error = target - self.current_estimate
self.current_estimate += 0.1 * error
return self.current_estimate, error
print("\nSimulating learning WITH and WITHOUT target networks...")
# Without target network
env1 = MovingTargetEnv()
estimates_no_target = []
errors_no_target = []
for i in range(100):
est, err = env1.step()
estimates_no_target.append(est)
errors_no_target.append(err)
# With target network (hard update every 10 steps)
env2 = MovingTargetEnv()
estimates_with_target = []
errors_with_target = []
target_estimate = 0.0
for i in range(100):
if i % 10 == 0: # Hard update
target_estimate = env2.current_estimate
est, err = env2.step_with_target(target_estimate)
estimates_with_target.append(est)
errors_with_target.append(err)
# Plot results
fig, axes = plt.subplots(2, 2, figsize=(15, 10))
# Plot 1: Estimates over time
ax = axes[0, 0]
ax.plot(estimates_no_target, 'r-', label='Without target network', alpha=0.7)
ax.plot(estimates_with_target, 'b-', label='With target network', alpha=0.7)
ax.axhline(y=5.0, color='green', linestyle='--', label='True value', alpha=0.5)
ax.set_xlabel('Update step')
ax.set_ylabel('Q-value estimate')
ax.set_title('Learning Stability Comparison')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 2: Errors over time
ax = axes[0, 1]
ax.plot(errors_no_target, 'r-', alpha=0.7, label='Without target')
ax.plot(errors_with_target, 'b-', alpha=0.7, label='With target')
ax.set_xlabel('Update step')
ax.set_ylabel('TD error')
ax.set_title('TD Error Comparison')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 3: Error distribution
ax = axes[1, 0]
ax.hist(errors_no_target, bins=20, alpha=0.7, color='red', label='Without target')
ax.hist(errors_with_target, bins=20, alpha=0.7, color='blue', label='With target')
ax.set_xlabel('TD error')
ax.set_ylabel('Frequency')
ax.set_title('Error Distribution')
ax.legend()
# Plot 4: Convergence speed
ax = axes[1, 1]
# Calculate convergence
def calculate_convergence(estimates, threshold=0.1):
"""Calculate steps to reach within threshold of true value"""
for i, est in enumerate(estimates):
if abs(est - 5.0) < threshold:
return i
return len(estimates)
conv_no_target = calculate_convergence(estimates_no_target)
conv_with_target = calculate_convergence(estimates_with_target)
bars = ax.bar(['Without target', 'With target'],
[conv_no_target, conv_with_target],
color=['red', 'blue'], alpha=0.7)
ax.set_ylabel('Steps to converge (within 0.1)')
ax.set_title('Convergence Speed')
# Add value labels
for bar, conv in zip(bars, [conv_no_target, conv_with_target]):
ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 2,
f'{conv}', ha='center', va='bottom')
plt.tight_layout()
plt.show()
print("\nKEY FINDINGS:")
print(f"1. Without target network: Converged in {conv_no_target} steps")
print(f"2. With target network: Converged in {conv_with_target} steps")
print(f"3. Target network reduced error variance by {np.var(errors_no_target)/np.var(errors_with_target):.1f}x")
print("\nPRACTICAL ADVICE:")
print("1. Use τ ≈ 0.001 for soft updates (slow tracking)")
print("2. Use 1000-10000 steps for hard updates")
print("3. Target networks essential for deep RL")
return estimates_no_target, estimates_with_target
# Run target network analysis
estimates_without, estimates_with = target_network_analysis()class DQNAgent:
"""Complete DQN agent implementation"""
def __init__(self, state_dim, action_dim, config=None):
"""
Initialize DQN agent
Args:
state_dim: Dimension of state space
action_dim: Number of possible actions
config: Configuration dictionary
"""
# Default configuration
self.config = {
'device': 'cuda' if torch.cuda.is_available() else 'cpu',
'gamma': 0.99, # Discount factor
'lr': 1e-4, # Learning rate
'buffer_size': 100000, # Replay buffer size
'batch_size': 64, # Training batch size
'update_target_every': 1000, # Hard update frequency
'tau': 0.001, # Soft update parameter
'epsilon_start': 1.0, # Initial exploration
'epsilon_end': 0.01, # Final exploration
'epsilon_decay': 0.995, # Exploration decay
'learning_starts': 10000, # Steps before learning
'train_frequency': 4, # Learn every N steps
}
if config:
self.config.update(config)
self.device = self.config['device']
self.action_dim = action_dim
self.state_dim = state_dim
# Initialize networks
self.policy_net = SimpleDQN(state_dim, 128, action_dim).to(self.device)
self.target_net = SimpleDQN(state_dim, 128, action_dim).to(self.device)
self.target_net.load_state_dict(self.policy_net.state_dict())
# Initialize optimizer
self.optimizer = optim.Adam(self.policy_net.parameters(), lr=self.config['lr'])
# Initialize replay buffer
self.memory = ReplayBuffer(self.config['buffer_size'], self.device)
# Training state
self.steps_done = 0
self.epsilon = self.config['epsilon_start']
self.episode_rewards = []
self.episode_lengths = []
self.losses = []
# For tracking
self.current_episode_reward = 0
self.current_episode_length = 0
def select_action(self, state, training=True):
"""
Select action using epsilon-greedy policy
Args:
state: Current state
training: Whether in training mode
Returns:
action: Integer action index
"""
if training and np.random.random() < self.epsilon:
# Explore
return np.random.randint(self.action_dim)
else:
# Exploit
with torch.no_grad():
state_tensor = torch.FloatTensor(state).unsqueeze(0).to(self.device)
q_values = self.policy_net(state_tensor)
return torch.argmax(q_values).item()
def store_transition(self, state, action, reward, next_state, done):
"""Store transition in replay buffer"""
state_tensor = torch.FloatTensor(state).unsqueeze(0).to(self.device)
next_state_tensor = torch.FloatTensor(next_state).unsqueeze(0).to(self.device)
self.memory.push(state_tensor, action, reward, next_state_tensor, done)
def update_epsilon(self):
"""Update exploration rate"""
self.epsilon = max(self.config['epsilon_end'],
self.config['epsilon_start'] *
self.config['epsilon_decay'] ** (self.steps_done / 1000))
def compute_td_loss(self, batch):
"""
Compute TD loss for a batch
Args:
batch: Tuple of (states, actions, rewards, next_states, dones)
Returns:
loss: Computed loss
"""
states, actions, rewards, next_states, dones = batch
# Current Q-values
current_q = self.policy_net(states).gather(1, actions)
# Next Q-values from target network
with torch.no_grad():
next_q = self.target_net(next_states).max(1)[0].unsqueeze(1)
# Compute target
target = rewards + self.config['gamma'] * next_q * (1 - dones)
# Compute loss
loss = F.smooth_l1_loss(current_q, target)
return loss
def learn(self):
"""Perform one learning step"""
if len(self.memory) < self.config['batch_size']:
return None
# Sample batch
batch = self.memory.sample(self.config['batch_size'])
# Compute loss
loss = self.compute_td_loss(batch)
# Optimize
self.optimizer.zero_grad()
loss.backward()
# Gradient clipping
torch.nn.utils.clip_grad_norm_(self.policy_net.parameters(), 10.0)
self.optimizer.step()
# Update target network
if self.steps_done % self.config['update_target_every'] == 0:
self.target_net.load_state_dict(self.policy_net.state_dict())
return loss.item()
def train_step(self, state, action, reward, next_state, done):
"""
Perform one training step (store + learn)
Returns:
loss: Loss value if learning happened, None otherwise
"""
# Store transition
self.store_transition(state, action, reward, next_state, done)
# Update counters
self.steps_done += 1
self.current_episode_reward += reward
self.current_episode_length += 1
# Update epsilon
self.update_epsilon()
# Learn if ready
loss = None
if (self.steps_done > self.config['learning_starts'] and
self.steps_done % self.config['train_frequency'] == 0):
loss = self.learn()
if loss is not None:
self.losses.append(loss)
# Handle episode end
if done:
self.episode_rewards.append(self.current_episode_reward)
self.episode_lengths.append(self.current_episode_length)
self.current_episode_reward = 0
self.current_episode_length = 0
return loss
def evaluate(self, env, num_episodes=10, render=False):
"""
Evaluate agent performance
Returns:
mean_reward: Average reward over episodes
"""
total_rewards = []
for episode in range(num_episodes):
state = env.reset()
done = False
episode_reward = 0
while not done:
if render:
env.render()
action = self.select_action(state, training=False)
state, reward, done = env.step(action)
episode_reward += reward
total_rewards.append(episode_reward)
mean_reward = np.mean(total_rewards)
std_reward = np.std(total_rewards)
return mean_reward, std_reward, total_rewards
def save(self, path):
"""Save agent state"""
torch.save({
'policy_net_state_dict': self.policy_net.state_dict(),
'target_net_state_dict': self.target_net.state_dict(),
'optimizer_state_dict': self.optimizer.state_dict(),
'steps_done': self.steps_done,
'epsilon': self.epsilon,
'episode_rewards': self.episode_rewards,
'losses': self.losses,
'config': self.config
}, path)
def load(self, path):
"""Load agent state"""
checkpoint = torch.load(path, map_location=self.device)
self.policy_net.load_state_dict(checkpoint['policy_net_state_dict'])
self.target_net.load_state_dict(checkpoint['target_net_state_dict'])
self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
self.steps_done = checkpoint['steps_done']
self.epsilon = checkpoint['epsilon']
self.episode_rewards = checkpoint['episode_rewards']
self.losses = checkpoint['losses']
self.config.update(checkpoint['config'])
def dqn_training_demo():
"""Complete DQN training demonstration"""
print("\n" + "="*70)
print("COMPLETE DQN TRAINING DEMONSTRATION")
print("="*70)
# Create a simple environment
class SimpleGridWorld:
"""Simple 10x10 grid world for DQN demo"""
def __init__(self, size=10):
self.size = size
self.goal = (size-1, size-1)
self.start = (0, 0)
self.obstacles = [(2, 2), (3, 5), (6, 7), (8, 3)]
# Actions: 0=up, 1=right, 2=down, 3=left
self.actions = [0, 1, 2, 3]
self.action_space = type('ActionSpace', (), {'n': 4})()
self.reset()
def reset(self):
"""Reset environment"""
self.position = list(self.start)
return self._get_state()
def step(self, action):
"""Take action"""
r, c = self.position
if action == 0: # up
r = max(0, r - 1)
elif action == 1: # right
c = min(self.size - 1, c + 1)
elif action == 2: # down
r = min(self.size - 1, r + 1)
elif action == 3: # left
c = max(0, c - 1)
# Check obstacle
if (r, c) in self.obstacles:
# Stay in place, penalty
reward = -5
done = False
else:
self.position = [r, c]
# Check goal
if (r, c) == self.goal:
reward = 10
done = True
else:
reward = -1 # Step penalty
done = False
return self._get_state(), reward, done
def _get_state(self):
"""Get state as one-hot encoded position"""
state = np.zeros(self.size * self.size)
r, c = self.position
idx = r * self.size + c
state[idx] = 1
return state
def render(self):
"""Simple text render"""
grid = [['.' for _ in range(self.size)] for _ in range(self.size)]
# Mark obstacles
for r, c in self.obstacles:
grid[r][c] = 'X'
# Mark goal
grid[self.goal[0]][self.goal[1]] = 'G'
# Mark agent
r, c = self.position
grid[r][c] = 'A'
print("\nGrid:")
for row in grid:
print(' '.join(row))
print()
print("\n1. INITIALIZING ENVIRONMENT AND AGENT...")
env = SimpleGridWorld(size=5) # Smaller for faster training
state_dim = env.size * env.size # 25 one-hot states
action_dim = env.action_space.n # 4 actions
print(f" State dimension: {state_dim} (one-hot position)")
print(f" Action dimension: {action_dim}")
print(f" Goal: Reach (4,4) from (0,0)")
print(f" Obstacles: {env.obstacles}")
# Create DQN agent
agent = DQNAgent(state_dim, action_dim, config={
'buffer_size': 10000,
'batch_size': 32,
'learning_starts': 1000,
'epsilon_decay': 0.999,
'update_target_every': 500
})
print("\n2. TRAINING AGENT...")
print(" (This will take a moment...)")
num_episodes = 500
print_progress_every = 50
for episode in range(num_episodes):
state = env.reset()
done = False
episode_losses = []
while not done:
# Select action
action = agent.select_action(state, training=True)
# Take action
next_state, reward, done = env.step(action)
# Train
loss = agent.train_step(state, action, reward, next_state, done)
if loss is not None:
episode_losses.append(loss)
state = next_state
# Print progress
if (episode + 1) % print_progress_every == 0:
avg_reward = np.mean(agent.episode_rewards[-print_progress_every:])
avg_length = np.mean(agent.episode_lengths[-print_progress_every:])
avg_loss = np.mean(episode_losses) if episode_losses else 0
print(f" Episode {episode + 1:4d} | "
f"Avg Reward: {avg_reward:6.2f} | "
f"Avg Length: {avg_length:5.1f} | "
f"Epsilon: {agent.epsilon:.3f} | "
f"Loss: {avg_loss:.4f}")
print("\n3. VISUALIZING RESULTS...")
# Create comprehensive visualization
fig, axes = plt.subplots(2, 3, figsize=(15, 10))
# Plot 1: Learning curve (rewards)
ax = axes[0, 0]
window = 20
if len(agent.episode_rewards) > window:
smoothed_rewards = np.convolve(agent.episode_rewards,
np.ones(window)/window,
mode='valid')
ax.plot(smoothed_rewards)
ax.set_xlabel('Episode')
ax.set_ylabel(f'Average Reward ({window}-episode window)')
ax.set_title('Learning Curve')
ax.grid(True, alpha=0.3)
# Mark best performance
best_idx = np.argmax(smoothed_rewards)
ax.axvline(x=best_idx, color='green', linestyle='--', alpha=0.5,
label=f'Best: {smoothed_rewards[best_idx]:.2f}')
ax.legend()
# Plot 2: Episode lengths
ax = axes[0, 1]
if len(agent.episode_lengths) > window:
smoothed_lengths = np.convolve(agent.episode_lengths,
np.ones(window)/window,
mode='valid')
ax.plot(smoothed_lengths)
ax.set_xlabel('Episode')
ax.set_ylabel(f'Average Length ({window}-episode window)')
ax.set_title('Episode Lengths')
ax.grid(True, alpha=0.3)
# Optimal path length (8 steps minimum)
ax.axhline(y=8, color='red', linestyle='--', alpha=0.5,
label='Optimal (8 steps)')
ax.legend()
# Plot 3: Loss over time
ax = axes[0, 2]
if agent.losses:
loss_window = max(1, len(agent.losses) // 100)
if loss_window > 1:
smoothed_losses = np.convolve(agent.losses,
np.ones(loss_window)/loss_window,
mode='valid')
ax.plot(smoothed_losses)
ax.set_xlabel('Training Step')
ax.set_ylabel(f'Smoothed Loss (window={loss_window})')
ax.set_title('Training Loss')
ax.set_yscale('log')
ax.grid(True, alpha=0.3)
# Plot 4: Epsilon decay
ax = axes[1, 0]
# Simulate epsilon decay for visualization
epsilons = []
steps = np.arange(0, agent.steps_done, 100)
for step in steps:
eps = max(agent.config['epsilon_end'],
agent.config['epsilon_start'] *
agent.config['epsilon_decay'] ** (step / 1000))
epsilons.append(eps)
ax.plot(steps, epsilons)
ax.set_xlabel('Training Steps')
ax.set_ylabel('Epsilon')
ax.set_title('Exploration Rate Decay')
ax.grid(True, alpha=0.3)
# Plot 5: Final policy visualization
ax = axes[1, 1]
# Create policy grid
policy_grid = np.zeros((env.size, env.size))
value_grid = np.zeros((env.size, env.size))
for r in range(env.size):
for c in range(env.size):
# Create state
state = np.zeros(state_dim)
idx = r * env.size + c
state[idx] = 1
# Get action and value
with torch.no_grad():
state_tensor = torch.FloatTensor(state).unsqueeze(0).to(agent.device)
q_values = agent.policy_net(state_tensor)
action = torch.argmax(q_values).item()
value = torch.max(q_values).item()
policy_grid[r, c] = action
value_grid[r, c] = value
# Show policy with arrows
action_symbols = {0: '↑', 1: '→', 2: '↓', 3: '←'}
im = ax.imshow(value_grid, cmap='viridis', alpha=0.7)
for r in range(env.size):
for c in range(env.size):
action = int(policy_grid[r, c])
symbol = action_symbols.get(action, '?')
ax.text(c, r, symbol, ha='center', va='center',
fontsize=12, fontweight='bold', color='white')
# Mark obstacles
for r, c in env.obstacles:
ax.add_patch(plt.Rectangle((c-0.5, r-0.5), 1, 1,
fill=False, edgecolor='red', linewidth=2))
# Mark start and goal
ax.text(env.start[1], env.start[0], 'S', ha='center', va='center',
fontsize=14, fontweight='bold', color='white')
ax.text(env.goal[1], env.goal[0], 'G', ha='center', va='center',
fontsize=14, fontweight='bold', color='white')
ax.set_title('Learned Policy and Values')
ax.set_xlabel('Column')
ax.set_ylabel('Row')
plt.colorbar(im, ax=ax, label='State Value')
# Plot 6: Q-value distribution
ax = axes[1, 2]
# Collect all Q-values
all_q_values = []
for r in range(env.size):
for c in range(env.size):
state = np.zeros(state_dim)
idx = r * env.size + c
state[idx] = 1
with torch.no_grad():
state_tensor = torch.FloatTensor(state).unsqueeze(0).to(agent.device)
q_values = agent.policy_net(state_tensor).cpu().numpy().flatten()
all_q_values.extend(q_values)
ax.hist(all_q_values, bins=30, alpha=0.7, edgecolor='black')
ax.set_xlabel('Q-value')
ax.set_ylabel('Frequency')
ax.set_title('Q-value Distribution')
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\n4. EVALUATING LEARNED AGENT...")
# Run evaluation
mean_reward, std_reward, eval_rewards = agent.evaluate(env, num_episodes=20)
print(f"\n Evaluation over 20 episodes:")
print(f" Average reward: {mean_reward:.2f} ± {std_reward:.2f}")
print(f" Best episode: {np.max(eval_rewards):.2f}")
print(f" Worst episode: {np.min(eval_rewards):.2f}")
# Success rate (positive reward means reached goal)
success_rate = sum(1 for r in eval_rewards if r > 0) / len(eval_rewards)
print(f" Success rate: {success_rate:.1%}")
print("\n5. DEMONSTRATING LEARNED BEHAVIOR...")
print("\n Running demonstration episode:")
state = env.reset()
done = False
total_reward = 0
steps = 0
positions = []
while not done and steps < 50:
action = agent.select_action(state, training=False)
state, reward, done = env.step(action)
# Decode position from one-hot
pos = np.argmax(state)
r = pos // env.size
c = pos % env.size
positions.append((r, c))
total_reward += reward
steps += 1
print(f" Total reward: {total_reward}")
print(f" Steps taken: {steps}")
print(f" Path: {positions[:10]}...") # Show first 10 positions
# Visualize path
plt.figure(figsize=(8, 8))
# Create grid
for r in range(env.size):
for c in range(env.size):
if (r, c) == env.start:
color = 'lightgreen'
elif (r, c) == env.goal:
color = 'lightcoral'
elif (r, c) in env.obstacles:
color = 'gray'
else:
color = 'white'
plt.gca().add_patch(plt.Rectangle((c-0.5, r-0.5), 1, 1,
facecolor=color, edgecolor='black'))
# Plot path
if positions:
positions = np.array(positions)
plt.plot(positions[:, 1], positions[:, 0], 'b-', linewidth=2, alpha=0.7)
plt.scatter(positions[:, 1], positions[:, 0], c=range(len(positions)),
cmap='viridis', s=50, zorder=5)
# Mark start and end
plt.scatter([env.start[1]], [env.start[0]], color='green',
s=200, marker='o', label='Start', zorder=10)
plt.scatter([env.goal[1]], [env.goal[0]], color='red',
s=200, marker='*', label='Goal', zorder=10)
plt.xlim(-0.5, env.size - 0.5)
plt.ylim(-0.5, env.size - 0.5)
plt.gca().invert_yaxis() # So (0,0) is top-left
plt.xlabel('Column')
plt.ylabel('Row')
plt.title('Learned Path Demonstration')
plt.legend()
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\n6. KEY INSIGHTS FROM DQN TRAINING:")
print("\n What worked well:")
print(" 1. Experience replay stabilized learning")
print(" 2. Target network prevented divergence")
print(" 3. Epsilon decay balanced exploration/exploitation")
print("\n Challenges observed:")
print(" 1. Needed sufficient exploration early on")
print(" 2. Learning rate crucial for convergence")
print(" 3. Buffer size affects sample diversity")
print("\n Improvements to try:")
print(" 1. Double DQN: Reduce overestimation bias")
print(" 2. Dueling DQN: Separate value and advantage")
print(" 3. Prioritized replay: Learn from important experiences")
return agent, env
# Run complete DQN training demo
dqn_agent, demo_env = dqn_training_demo()class AdvancedDQNVariants:
"""Implement advanced DQN variants"""
@staticmethod
def double_dqn_loss(policy_net, target_net, batch, gamma=0.99):
"""
Double DQN loss calculation
Reduces overestimation bias by using policy network
to select actions, target network to evaluate them
"""
states, actions, rewards, next_states, dones = batch
# Current Q-values
current_q = policy_net(states).gather(1, actions)
# Double DQN: use policy net to select actions
with torch.no_grad():
# Select actions using policy network
next_actions = policy_net(next_states).argmax(1, keepdim=True)
# Evaluate using target network
next_q = target_net(next_states).gather(1, next_actions)
# Compute target
target = rewards + gamma * next_q * (1 - dones)
# Compute loss
loss = F.smooth_l1_loss(current_q, target)
return loss
@staticmethod
def dueling_dqn_network(input_dim, hidden_dim, n_actions):
"""
Dueling DQN network architecture
Separates value and advantage streams:
Q(s,a) = V(s) + A(s,a) - mean(A(s,:))
"""
class DuelingDQN(nn.Module):
def __init__(self, input_dim, hidden_dim, n_actions):
super(DuelingDQN, self).__init__()
# Shared feature layer
self.feature = nn.Sequential(
nn.Linear(input_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim),
nn.ReLU()
)
# Value stream
self.value_stream = nn.Sequential(
nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, 1)
)
# Advantage stream
self.advantage_stream = nn.Sequential(
nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, n_actions)
)
def forward(self, x):
features = self.feature(x)
value = self.value_stream(features)
advantage = self.advantage_stream(features)
# Combine: Q(s,a) = V(s) + A(s,a) - mean(A(s,:))
q_values = value + advantage - advantage.mean(1, keepdim=True)
return q_values
return DuelingDQN(input_dim, hidden_dim, n_actions)
@staticmethod
def noisy_linear(in_features, out_features, sigma_init=0.5):
"""
Noisy linear layer for exploration
Uses parameterized noise for exploration
instead of epsilon-greedy
"""
class NoisyLinear(nn.Module):
def __init__(self, in_features, out_features, sigma_init):
super(NoisyLinear, self).__init__()
self.in_features = in_features
self.out_features = out_features
# Learnable parameters
self.mu_weight = nn.Parameter(torch.Tensor(out_features, in_features))
self.sigma_weight = nn.Parameter(torch.Tensor(out_features, in_features))
self.mu_bias = nn.Parameter(torch.Tensor(out_features))
self.sigma_bias = nn.Parameter(torch.Tensor(out_features))
# Register noise buffers
self.register_buffer('epsilon_weight', torch.Tensor(out_features, in_features))
self.register_buffer('epsilon_bias', torch.Tensor(out_features))
# Initialize
self.reset_parameters(sigma_init)
self.reset_noise()
def reset_parameters(self, sigma_init):
# Initialize mu weights (like normal linear layer)
bound = 1 / np.sqrt(self.in_features)
self.mu_weight.data.uniform_(-bound, bound)
self.mu_bias.data.uniform_(-bound, bound)
# Initialize sigma weights
self.sigma_weight.data.fill_(sigma_init / np.sqrt(self.in_features))
self.sigma_bias.data.fill_(sigma_init / np.sqrt(self.out_features))
def reset_noise(self):
# Sample new noise
epsilon_in = self._scale_noise(self.in_features)
epsilon_out = self._scale_noise(self.out_features)
self.epsilon_weight.copy_(epsilon_out.ger(epsilon_in))
self.epsilon_bias.copy_(epsilon_out)
def _scale_noise(self, size):
x = torch.randn(size)
return x.sign().mul(x.abs().sqrt())
def forward(self, x):
# Noisy weights
weight = self.mu_weight + self.sigma_weight * self.epsilon_weight
bias = self.mu_bias + self.sigma_bias * self.epsilon_bias
return F.linear(x, weight, bias)
return NoisyLinear(in_features, out_features, sigma_init)
@staticmethod
def multi_step_dqn_loss(policy_net, target_net, batch, gamma=0.99, n_steps=3):
"""
Multi-step DQN loss
Uses n-step returns instead of 1-step returns
"""
states, actions, rewards, next_states, dones = batch
# Current Q-values
current_q = policy_net(states).gather(1, actions)
# Compute n-step returns
with torch.no_grad():
# For simplicity, assume batch contains n-step transitions
# In practice, you'd need to store n-step transitions in buffer
n_step_returns = torch.zeros_like(rewards)
# Bootstrap with target network
final_next_q = target_net(next_states).max(1, keepdim=True)[0]
# Compute n-step return
cumulative_gamma = gamma ** n_steps
n_step_returns = rewards.clone()
for i in range(1, n_steps):
n_step_returns += (gamma ** i) * rewards.roll(-i, dims=0)
# Handle episode boundaries
mask = (dones.roll(-i, dims=0) == 0).float()
n_step_returns *= mask
n_step_returns += cumulative_gamma * final_next_q * (1 - dones)
# Compute loss
loss = F.smooth_l1_loss(current_q, n_step_returns)
return loss
def advanced_dqn_comparison():
"""Compare advanced DQN variants"""
print("\n" + "="*70)
print("ADVANCED DQN VARIANTS COMPARISON")
print("="*70)
print("\n1. DOUBLE DQN:")
print(" Problem: Q-learning overestimates Q-values")
print(" Solution: Use different networks for action selection and evaluation")
print(" Formula: target = r + γ Q_target(s', argmax_a Q(s',a))")
print("\n2. DUELING DQN:")
print(" Insight: Value of state and advantage of action can be separated")
print(" Architecture: Two streams → V(s) and A(s,a)")
print(" Formula: Q(s,a) = V(s) + A(s,a) - mean_a A(s,a)")
print("\n3. PRIORITIZED EXPERIENCE REPLAY:")
print(" Insight: Not all experiences are equally important")
print(" Solution: Sample transitions with probability ∝ TD error")
print(" Benefit: Faster learning, focus on surprising experiences")
print("\n4. MULTI-STEP LEARNING:")
print(" Problem: 1-step TD has high bias")
print(" Solution: Use n-step returns")
print(" Tradeoff: Between TD (bias) and MC (variance)")
print("\n5. NOISY NETS:")
print(" Problem: Epsilon-greedy explores randomly")
print(" Solution: Add parameterized noise to weights")
print(" Benefit: State-dependent exploration")
print("\n6. DISTRIBUTIONAL DQN:")
print(" Insight: Learn distribution of returns, not just expectation")
print(" Architecture: Output probability distribution over returns")
print(" Benefit: Better risk assessment, richer learning signal")
print("\n7. RAINBOW DQN:")
print(" Combines all 7 improvements:")
print(" 1. Double DQN")
print(" 2. Dueling architecture")
print(" 3. Prioritized replay")
print(" 4. Multi-step learning")
print(" 5. Noisy nets")
print(" 6. Distributional RL")
print(" 7. N-step returns")
# Create a simple comparison
class ComparisonEnv:
def __init__(self):
self.n_states = 20
self.actions = [0, 1]
self.action_space = type('ActionSpace', (), {'n': 2})()
def reset(self):
return np.random.randint(self.n_states)
def step(self, action):
state = np.random.randint(self.n_states)
reward = np.sin(state/self.n_states * 2*np.pi) + np.random.normal(0, 0.1)
done = np.random.random() < 0.1
return state, reward, done
env = ComparisonEnv()
state_dim = 20 # One-hot encoding
action_dim = 2
# Test different variants
variants = {
'Vanilla DQN': SimpleDQN(state_dim, 64, action_dim),
'Double DQN': SimpleDQN(state_dim, 64, action_dim),
'Dueling DQN': AdvancedDQNVariants.dueling_dqn_network(state_dim, 64, action_dim)
}
# Simple training comparison
num_episodes = 200
results = {name: [] for name in variants}
for name, network in variants.items():
print(f"\nTraining {name}...")
optimizer = optim.Adam(network.parameters(), lr=0.001)
buffer = ReplayBuffer(1000, device='cpu')
episode_rewards = []
for episode in range(num_episodes):
state = env.reset()
state_onehot = np.zeros(state_dim)
state_onehot[state] = 1
done = False
episode_reward = 0
while not done:
# Select action
state_tensor = torch.FloatTensor(state_onehot).unsqueeze(0)
q_values = network(state_tensor)
action = torch.argmax(q_values).item() if np.random.random() > 0.1 else np.random.randint(2)
# Take step
next_state, reward, done = env.step(action)
next_state_onehot = np.zeros(state_dim)
next_state_onehot[next_state] = 1
# Store
buffer.push(torch.FloatTensor(state_onehot).unsqueeze(0),
action, reward,
torch.FloatTensor(next_state_onehot).unsqueeze(0),
done)
# Learn if buffer has enough samples
if len(buffer) >= 32:
batch = buffer.sample(32)
states, actions, rewards, next_states, dones = batch
# Compute loss based on variant
if name == 'Vanilla DQN' or name == 'Dueling DQN':
current_q = network(states).gather(1, actions)
with torch.no_grad():
next_q = network(next_states).max(1)[0].unsqueeze(1)
target = rewards + 0.99 * next_q * (1 - dones)
elif name == 'Double DQN':
# For Double DQN, we'd use a target network in practice
# This is a simplified version
current_q = network(states).gather(1, actions)
with torch.no_grad():
next_actions = network(next_states).argmax(1, keepdim=True)
next_q = network(next_states).gather(1, next_actions)
target = rewards + 0.99 * next_q * (1 - dones)
loss = F.mse_loss(current_q, target)
# Optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
episode_reward += reward
state_onehot = next_state_onehot
episode_rewards.append(episode_reward)
results[name] = episode_rewards
# Plot comparison
plt.figure(figsize=(12, 8))
# Plot 1: Learning curves
plt.subplot(2, 2, 1)
window = 10
for name, rewards in results.items():
if len(rewards) > window:
smoothed = np.convolve(rewards, np.ones(window)/window, mode='valid')
plt.plot(smoothed, label=name, linewidth=2)
plt.xlabel('Episode')
plt.ylabel(f'Average Reward ({window}-episode window)')
plt.title('Learning Curve Comparison')
plt.legend()
plt.grid(True, alpha=0.3)
# Plot 2: Final performance
plt.subplot(2, 2, 2)
final_performance = {name: np.mean(rewards[-20:]) for name, rewards in results.items()}
names = list(final_performance.keys())
values = list(final_performance.values())
bars = plt.bar(names, values, alpha=0.7)
plt.ylabel('Average Reward (last 20 episodes)')
plt.title('Final Performance')
plt.xticks(rotation=15)
# Add value labels
for bar, value in zip(bars, values):
plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.01,
f'{value:.3f}', ha='center', va='bottom')
# Plot 3: Architecture visualization
plt.subplot(2, 2, 3)
# Draw simple architecture diagrams
architectures = {
'Vanilla DQN': 'State → [FC] → [FC] → Q-values',
'Double DQN': 'Two networks\nSelect: Policy net\nEvaluate: Target net',
'Dueling DQN': 'State → Shared → [Value] + [Advantage] → Q-values'
}
y_pos = 0.8
for name, desc in architectures.items():
plt.text(0.1, y_pos, f'{name}:', fontweight='bold', fontsize=10)
plt.text(0.2, y_pos - 0.1, desc, fontsize=9)
y_pos -= 0.25
plt.xlim(0, 1)
plt.ylim(0, 1)
plt.axis('off')
plt.title('Architecture Comparison')
# Plot 4: Relative improvement
plt.subplot(2, 2, 4)
baseline = final_performance['Vanilla DQN']
improvements = {}
for name, value in final_performance.items():
if name != 'Vanilla DQN':
improvement = (value - baseline) / abs(baseline) * 100
improvements[name] = improvement
if improvements:
names = list(improvements.keys())
values = list(improvements.values())
colors = ['green' if v > 0 else 'red' for v in values]
bars = plt.bar(names, values, color=colors, alpha=0.7)
plt.ylabel('Improvement over Vanilla DQN (%)')
plt.title('Relative Performance Gain')
plt.axhline(y=0, color='black', linestyle='-', linewidth=0.5)
# Add value labels
for bar, value in zip(bars, values):
plt.text(bar.get_x() + bar.get_width()/2,
bar.get_height() + (1 if value > 0 else -3),
f'{value:+.1f}%', ha='center', va='bottom' if value > 0 else 'top')
plt.tight_layout()
plt.show()
print("\nKEY TAKEAWAYS:")
print("\n1. Vanilla DQN:")
print(" - Foundation, but has limitations")
print(" - Suffers from overestimation bias")
print("\n2. Double DQN:")
print(" - Simple but effective improvement")
print(" - Reduces overestimation significantly")
print("\n3. Dueling DQN:")
print(" - Better value function learning")
print(" - Especially helpful when many actions have similar values")
print("\n4. Rainbow DQN:")
print(" - State-of-the-art for discrete actions")
print(" - Combines all improvements synergistically")
print("\nRECOMMENDATIONS:")
print("1. Start with Double DQN (easiest improvement)")
print("2. Add Dueling architecture for complex environments")
print("3. Use Prioritized Replay for sample efficiency")
print("4. Try Rainbow for maximum performance")
return results
# Run advanced DQN comparison
advanced_results = advanced_dqn_comparison()Chapter 8 Key Takeaways:
-
The DQN Breakthrough (2013-2015):
- First to learn directly from pixels
- Human-level performance on Atari games
- Key innovations: Experience replay + Target networks
-
Neural Networks as Function Approximators:
- Can handle high-dimensional inputs
- Learn features automatically (no manual engineering)
- Universal function approximators
-
Experience Replay:
- Stores (s, a, r, s', done) in circular buffer
- Samples random mini-batches
- Breaks temporal correlations
- Increases data efficiency
-
Target Networks:
- Separate network for computing targets
- Updated slowly (hard or soft updates)
- Stabilizes learning by fixing targets temporarily
-
Complete DQN Algorithm:
Initialize replay buffer D, Q-network θ, target network θ' for episode = 1 to M: Initialize state s for t = 1 to T: With probability ε select random action a Otherwise a = argmax_a Q(s,a; θ) Execute a, observe r, s' Store (s,a,r,s',done) in D Sample random batch from D Compute targets using θ' Update θ with gradient descent Every C steps: θ' ← θ -
Advanced Variants:
- Double DQN: Reduces overestimation bias
- Dueling DQN: Separates value and advantage
- Prioritized Replay: Samples important transitions
- Multi-step Learning: Uses n-step returns
- Noisy Nets: Parameterized noise for exploration
- Distributional DQN: Learns value distributions
- Rainbow DQN: Combines all improvements
-
Practical Implementation Tips:
- Start with simple environments
- Monitor loss and reward curves
- Tune learning rate and exploration schedule
- Use gradient clipping for stability
- Visualize learned policies and value functions
Analogy: The Sculptor vs The Architect
- Value-based methods (DQN): Like an architect who calculates the value of each brick, then decides where to place them
- Policy gradients: Like a sculptor who directly shapes the clay, adjusting based on the final sculpture's beauty
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import numpy as np
import matplotlib.pyplot as plt
from collections import deque, namedtuple
import random
import gym
class PolicyGradientIntroduction:
"""Introduction to Policy Gradient Methods"""
def __init__(self):
self.explain_fundamentals()
def explain_fundamentals(self):
"""Explain the fundamental concepts"""
print("="*70)
print("POLICY GRADIENTS: DIRECT ACTION OPTIMIZATION")
print("="*70)
print("\nTHE PHILOSOPHICAL DIFFERENCE:")
print("\nValue-Based Methods (DQN, Q-learning):")
print("1. Learn value function Q(s,a)")
print("2. Policy is implicit: π(s) = argmax_a Q(s,a)")
print("3. Works well for discrete actions")
print("\nPolicy-Based Methods (Policy Gradients):")
print("1. Directly learn policy π_θ(a|s)")
print("2. Outputs action probabilities")
print("3. Can handle continuous actions naturally")
print("\n" + "-"*70)
print("WHEN TO USE POLICY GRADIENTS:")
print("-"*70)
scenarios = [
("Continuous action spaces", "Robotics, physics control"),
("Stochastic policies needed", "Game theory, adversarial settings"),
("Natural exploration", "Learns distribution, not just greedy"),
("Simplicity", "Direct optimization of what we care about")
]
for scenario, example in scenarios:
print(f"✓ {scenario:30s}: {example}")
print("\n" + "-"*70)
print("THE CORE IDEA:")
print("-"*70)
print("\nInstead of: Learn values → Derive policy")
print("We do: Directly optimize policy parameters θ")
print("\nObjective: Maximize expected return J(θ) = E[Σγ^t r_t]")
print("Update: θ ← θ + α ∇_θ J(θ)")
# Mathematical formulation
print("\n" + "-"*70)
print("MATHEMATICAL FORMULATION:")
print("-"*70)
equations = [
("Policy", "π_θ(a|s) = probability of action a in state s"),
("Objective", "J(θ) = E_{τ∼π_θ}[Σ_{t=0}^T γ^t r_t]"),
("Gradient", "∇_θ J(θ) = E_{τ∼π_θ}[Σ_t ∇_θ log π_θ(a_t|s_t) G_t]"),
("Update", "θ ← θ + α ∇_θ J(θ)")
]
for name, eq in equations:
print(f"\n{name:12s}: {eq}")
def visualize_policy_vs_value(self):
"""Visualize difference between policy and value functions"""
# Create a simple 1D environment
states = np.linspace(-2, 2, 100)
# Value function (hypothetical)
values = -states**2 + 2
# Policy (hypothetical)
# Mean action increases with state
policy_means = np.tanh(states)
policy_stds = 0.3 * np.ones_like(states)
# Plot comparison
fig, axes = plt.subplots(1, 2, figsize=(12, 5))
# Value function plot
ax = axes[0]
ax.plot(states, values, 'b-', linewidth=3, label='V(s)')
ax.set_xlabel('State s')
ax.set_ylabel('Value V(s)')
ax.set_title('Value-Based Approach')
ax.grid(True, alpha=0.3)
ax.legend()
# Add greedy policy arrows
for s, v in zip(states[::20], values[::20]):
# Greedy action: move toward higher value
ax.arrow(s, v-0.1, 0.1 if s < 0 else -0.1, 0,
head_width=0.05, head_length=0.1, fc='red', ec='red', alpha=0.5)
# Policy plot
ax = axes[1]
# Show policy as distribution
for s, mean, std in zip(states[::10], policy_means[::10], policy_stds[::10]):
# Plot Gaussian distribution
actions = np.linspace(-2, 2, 50)
pdf = np.exp(-0.5 * ((actions - mean) / std) ** 2) / (std * np.sqrt(2 * np.pi))
ax.plot(actions, pdf * 0.5 + s/5, 'b-', alpha=0.5)
ax.set_xlabel('Action a')
ax.set_ylabel('State s (offset)')
ax.set_title('Policy-Based Approach: π(a|s)')
ax.grid(True, alpha=0.3)
# Add mean action line
ax.plot(policy_means, states/5, 'r--', linewidth=2, label='Mean action')
ax.legend()
plt.tight_layout()
plt.show()
print("\nVISUAL INTERPRETATION:")
print("Left: Value function - learn value, derive greedy policy")
print("Right: Policy - directly learn action probabilities")
# Initialize introduction
pg_intro = PolicyGradientIntroduction()
pg_intro.visualize_policy_vs_value()class PolicyGradientTheorem:
"""Understanding and implementing the Policy Gradient Theorem"""
def __init__(self):
self.explain_theorem()
def explain_theorem(self):
"""Explain the Policy Gradient Theorem"""
print("="*70)
print("THE POLICY GRADIENT THEOREM")
print("="*70)
print("\nTHE PROBLEM:")
print("How to compute ∇_θ J(θ) when J(θ) depends on:")
print("1. Action selection (depends on θ)")
print("2. State distribution (depends on θ)")
print("3. Rewards (depend on environment)")
print("\n" + "-"*70)
print("THE BREAKTHROUGH:")
print("-"*70)
print("\nPolicy Gradient Theorem (Sutton et al., 2000):")
print("∇_θ J(θ) = E_{τ∼π_θ}[Σ_t ∇_θ log π_θ(a_t|s_t) G_t]")
print("\nWhere:")
print("• τ = (s_0, a_0, r_0, s_1, a_1, r_1, ...) is a trajectory")
print("• G_t = Σ_{k=t}^T γ^{k-t} r_k is the return from time t")
print("• π_θ(a_t|s_t) is the policy probability")
print("• ∇_θ log π_θ(a_t|s_t) is the score function")
print("\n" + "-"*70)
print("WHY IT'S AMAZING:")
print("-"*70)
amazing_facts = [
("No need for model", "Works in unknown environments"),
("Handles continuous actions", "Natural for robotics"),
("Stochastic policies", "Exploration built-in"),
("Simple implementation", "Just need log probabilities")
]
for fact, reason in amazing_facts:
print(f"✓ {fact:25s}: {reason}")
def derive_theorem(self):
"""Step-by-step derivation of the theorem"""
print("\n" + "="*70)
print("STEP-BY-STEP DERIVATION")
print("="*70)
steps = [
("1. Objective", "J(θ) = Σ_τ P(τ|θ) R(τ)"),
("2. Gradient", "∇_θ J(θ) = Σ_τ ∇_θ P(τ|θ) R(τ)"),
("3. Log trick", "∇_θ P(τ|θ) = P(τ|θ) ∇_θ log P(τ|θ)"),
("4. Trajectory prob", "P(τ|θ) = Π_t π_θ(a_t|s_t) P(s_{t+1}|s_t,a_t)"),
("5. Log trajectory", "log P(τ|θ) = Σ_t [log π_θ(a_t|s_t) + log P(s_{t+1}|s_t,a_t)]"),
("6. Gradient", "∇_θ log P(τ|θ) = Σ_t ∇_θ log π_θ(a_t|s_t)"),
("7. Final form", "∇_θ J(θ) = E_τ[Σ_t ∇_θ log π_θ(a_t|s_t) R(τ)]"),
("8. Causality", "∇_θ J(θ) = E_τ[Σ_t ∇_θ log π_θ(a_t|s_t) G_t]")
]
for step, equation in steps:
print(f"\n{step:20s}: {equation}")
print("\nKEY INSIGHTS:")
print("1. Environment dynamics P(s'|s,a) don't depend on θ")
print("2. Only need log probabilities of actions")
print("3. Future rewards don't affect past action gradients")
def simple_implementation(self):
"""Simple implementation of policy gradient"""
print("\n" + "="*70)
print("SIMPLE IMPLEMENTATION")
print("="*70)
class SimplePolicyNetwork(nn.Module):
"""Simple policy network for discrete actions"""
def __init__(self, input_dim, hidden_dim, output_dim):
super().__init__()
self.net = nn.Sequential(
nn.Linear(input_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, output_dim)
)
def forward(self, x):
return self.net(x)
def get_action(self, state):
"""Sample action from policy"""
logits = self.forward(state)
probs = F.softmax(logits, dim=-1)
dist = torch.distributions.Categorical(probs)
action = dist.sample()
log_prob = dist.log_prob(action)
return action.item(), log_prob
# Create environment
class SimpleEnv:
def __init__(self):
self.state = 0
self.states = [0, 1, 2]
self.actions = [0, 1]
self.transitions = {
0: {0: (1, 1), 1: (2, -1)}, # State 0
1: {0: (2, 2), 1: (0, 0)}, # State 1
2: {0: (0, -1), 1: (1, 1)} # State 2
}
def reset(self):
self.state = 0
return torch.tensor([self.state], dtype=torch.float32)
def step(self, action):
next_state, reward = self.transitions[self.state][action]
self.state = next_state
done = (self.state == 2) # Terminal state
return (torch.tensor([self.state], dtype=torch.float32),
reward, done)
# Initialize
env = SimpleEnv()
policy = SimplePolicyNetwork(1, 32, 2)
optimizer = optim.Adam(policy.parameters(), lr=0.01)
print("\nTraining loop pseudocode:")
print("""
for episode in range(num_episodes):
state = env.reset()
rewards = []
log_probs = []
while not done:
action, log_prob = policy.get_action(state)
next_state, reward, done = env.step(action)
rewards.append(reward)
log_probs.append(log_prob)
state = next_state
# Compute returns
returns = compute_returns(rewards, gamma)
# Compute loss
policy_loss = []
for log_prob, G in zip(log_probs, returns):
policy_loss.append(-log_prob * G) # Negative for gradient ascent
loss = torch.stack(policy_loss).sum()
# Optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
""")
return policy, env
# Run theorem explanation
pg_theorem = PolicyGradientTheorem()
pg_theorem.derive_theorem()
simple_policy, simple_env = pg_theorem.simple_implementation()class REINFORCEAgent:
"""REINFORCE: Monte Carlo Policy Gradient"""
def __init__(self, state_dim, action_dim, hidden_dim=128, gamma=0.99, lr=0.001):
"""
Initialize REINFORCE agent
Args:
state_dim: Dimension of state space
action_dim: Number of possible actions
hidden_dim: Size of hidden layers
gamma: Discount factor
lr: Learning rate
"""
self.state_dim = state_dim
self.action_dim = action_dim
self.gamma = gamma
# Policy network
self.policy_net = self._build_policy_network(state_dim, hidden_dim, action_dim)
self.optimizer = optim.Adam(self.policy_net.parameters(), lr=lr)
# Training history
self.episode_rewards = []
self.episode_lengths = []
self.losses = []
def _build_policy_network(self, state_dim, hidden_dim, action_dim):
"""Build policy network for discrete actions"""
class PolicyNetwork(nn.Module):
def __init__(self, state_dim, hidden_dim, action_dim):
super().__init__()
self.net = nn.Sequential(
nn.Linear(state_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, action_dim)
)
def forward(self, x):
return self.net(x)
def get_action(self, state):
"""Sample action and get log probability"""
logits = self.forward(state)
probs = F.softmax(logits, dim=-1)
dist = torch.distributions.Categorical(probs)
action = dist.sample()
log_prob = dist.log_prob(action)
return action.item(), log_prob, probs.detach()
return PolicyNetwork(state_dim, hidden_dim, action_dim)
def select_action(self, state, training=True):
"""
Select action using current policy
Args:
state: Current state
training: Whether to use stochastic policy
Returns:
action: Selected action
log_prob: Log probability of action (if training)
probs: Action probabilities (for analysis)
"""
state_tensor = torch.FloatTensor(state).unsqueeze(0)
if training:
action, log_prob, probs = self.policy_net.get_action(state_tensor)
return action, log_prob, probs
else:
with torch.no_grad():
logits = self.policy_net(state_tensor)
probs = F.softmax(logits, dim=-1)
action = torch.argmax(probs).item()
return action, None, probs
def compute_returns(self, rewards):
"""
Compute discounted returns from rewards
Args:
rewards: List of rewards from episode
Returns:
returns: Discounted returns for each time step
"""
returns = []
G = 0
for r in reversed(rewards):
G = r + self.gamma * G
returns.insert(0, G)
return returns
def update_policy(self, log_probs, returns):
"""
Update policy using REINFORCE
Args:
log_probs: List of log probabilities from episode
returns: List of returns from episode
Returns:
loss: Computed loss value
"""
# Convert to tensors
log_probs = torch.stack(log_probs)
returns = torch.FloatTensor(returns)
# Normalize returns (reduces variance)
returns = (returns - returns.mean()) / (returns.std() + 1e-8)
# Compute loss: -Σ log_prob * G
policy_loss = []
for log_prob, G in zip(log_probs, returns):
policy_loss.append(-log_prob * G)
loss = torch.stack(policy_loss).sum()
# Optimize
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
return loss.item()
def train_episode(self, env, max_steps=1000, render=False):
"""
Train for one episode
Returns:
episode_reward: Total reward for episode
episode_length: Length of episode
loss: Loss value if training happened
"""
state = env.reset()
done = False
episode_reward = 0
episode_length = 0
# Store episode data
rewards = []
log_probs = []
while not done and episode_length < max_steps:
if render:
env.render()
# Select action
action, log_prob, _ = self.select_action(state, training=True)
# Take action
next_state, reward, done = env.step(action)
# Store data
rewards.append(reward)
log_probs.append(log_prob)
# Update
episode_reward += reward
episode_length += 1
state = next_state
# Compute returns and update policy
returns = self.compute_returns(rewards)
loss = self.update_policy(log_probs, returns)
# Store history
self.episode_rewards.append(episode_reward)
self.episode_lengths.append(episode_length)
self.losses.append(loss)
return episode_reward, episode_length, loss
def evaluate(self, env, num_episodes=10, render=False):
"""
Evaluate agent performance
Returns:
mean_reward: Average reward
std_reward: Standard deviation
all_rewards: List of all rewards
"""
rewards = []
for episode in range(num_episodes):
state = env.reset()
done = False
episode_reward = 0
if render:
print(f"\nEvaluation Episode {episode + 1}:")
env.render()
while not done:
action, _, probs = self.select_action(state, training=False)
state, reward, done = env.step(action)
episode_reward += reward
if render:
env.render()
print(f"Action: {action}, Reward: {reward}, Total: {episode_reward}")
rewards.append(episode_reward)
mean_reward = np.mean(rewards)
std_reward = np.std(rewards)
return mean_reward, std_reward, rewards
def visualize_training(self):
"""Visualize training progress"""
if not self.episode_rewards:
print("No training data to visualize")
return
fig, axes = plt.subplots(2, 3, figsize=(15, 10))
# Plot 1: Learning curve
ax = axes[0, 0]
window = min(50, len(self.episode_rewards) // 10)
if window > 1:
smoothed_rewards = np.convolve(self.episode_rewards,
np.ones(window)/window,
mode='valid')
ax.plot(smoothed_rewards)
ax.set_xlabel('Episode')
ax.set_ylabel(f'Average Reward ({window}-episode window)')
ax.set_title('Learning Curve')
ax.grid(True, alpha=0.3)
# Mark best performance
best_idx = np.argmax(smoothed_rewards)
ax.axvline(x=best_idx, color='green', linestyle='--', alpha=0.5,
label=f'Best: {smoothed_rewards[best_idx]:.2f}')
ax.legend()
# Plot 2: Episode lengths
ax = axes[0, 1]
if window > 1:
smoothed_lengths = np.convolve(self.episode_lengths,
np.ones(window)/window,
mode='valid')
ax.plot(smoothed_lengths)
ax.set_xlabel('Episode')
ax.set_ylabel(f'Average Length ({window}-episode window)')
ax.set_title('Episode Lengths')
ax.grid(True, alpha=0.3)
# Plot 3: Loss over time
ax = axes[0, 2]
if self.losses:
loss_window = max(1, len(self.losses) // 50)
if loss_window > 1:
smoothed_losses = np.convolve(self.losses,
np.ones(loss_window)/loss_window,
mode='valid')
ax.plot(smoothed_losses)
ax.set_xlabel('Episode')
ax.set_ylabel(f'Smoothed Loss (window={loss_window})')
ax.set_title('Training Loss')
ax.set_yscale('log')
ax.grid(True, alpha=0.3)
# Plot 4: Reward distribution
ax = axes[1, 0]
if len(self.episode_rewards) > 10:
ax.hist(self.episode_rewards[-100:], bins=20, alpha=0.7, edgecolor='black')
ax.set_xlabel('Episode Reward')
ax.set_ylabel('Frequency')
ax.set_title('Recent Reward Distribution')
ax.grid(True, alpha=0.3)
# Plot 5: Policy visualization (if we have a simple env)
ax = axes[1, 1]
# This would be environment-specific
ax.text(0.5, 0.5, 'Policy visualization\n(Environment specific)',
ha='center', va='center', transform=ax.transAxes, fontsize=12)
ax.set_title('Learned Policy')
ax.axis('off')
# Plot 6: Gradient flow analysis
ax = axes[1, 2]
if self.losses:
# Compute gradient magnitudes (simplified)
grad_norms = []
for i in range(0, len(self.losses), 10):
if i < len(self.losses):
# Simplified: loss change as proxy for gradient
if i > 0:
grad_norms.append(abs(self.losses[i] - self.losses[i-1]))
if grad_norms:
ax.plot(grad_norms)
ax.set_xlabel('Training Step (every 10th)')
ax.set_ylabel('Approximate Gradient Norm')
ax.set_title('Gradient Magnitude Over Time')
ax.set_yscale('log')
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
def reinforce_demo():
"""Demonstrate REINFORCE algorithm"""
print("="*70)
print("REINFORCE DEMONSTRATION")
print("="*70)
# Create a simple environment
class SimpleREnv:
"""Simple environment for REINFORCE demo"""
def __init__(self, n_states=5):
self.n_states = n_states
self.goal = n_states - 1
self.state = 0
# Actions: 0=left, 1=right
self.actions = [0, 1]
self.action_space = type('ActionSpace', (), {'n': 2})()
def reset(self):
self.state = 0
return self._get_state()
def step(self, action):
if action == 0: # left
self.state = max(0, self.state - 1)
else: # right
self.state = min(self.n_states - 1, self.state + 1)
# Rewards
if self.state == self.goal:
reward = 10.0
done = True
else:
reward = -0.1 # Small penalty for each step
done = False
return self._get_state(), reward, done
def _get_state(self):
"""One-hot encoded state"""
state = np.zeros(self.n_states)
state[self.state] = 1
return state
def render(self):
"""Simple visualization"""
line = ['_'] * self.n_states
line[self.state] = 'A'
line[self.goal] = 'G'
print('[' + ' '.join(line) + ']')
print("\n1. ENVIRONMENT SETUP:")
print(" Simple chain environment:")
print(" States: 0-4, Goal: state 4")
print(" Actions: 0=left, 1=right")
print(" Reward: +10 at goal, -0.1 per step")
env = SimpleREnv(n_states=5)
print("\n2. AGENT INITIALIZATION:")
print(" Using REINFORCE (Monte Carlo Policy Gradient)")
print(" Policy network: 5 → 64 → 64 → 2")
print(" Learning rate: 0.01, Gamma: 0.99")
agent = REINFORCEAgent(
state_dim=5, # One-hot encoded states
action_dim=2,
hidden_dim=64,
gamma=0.99,
lr=0.01
)
print("\n3. TRAINING AGENT...")
print(" (Training for 500 episodes)")
num_episodes = 500
print_every = 50
for episode in range(num_episodes):
episode_reward, episode_length, loss = agent.train_episode(env, max_steps=100)
if (episode + 1) % print_every == 0:
avg_reward = np.mean(agent.episode_rewards[-print_every:])
avg_length = np.mean(agent.episode_lengths[-print_every:])
print(f" Episode {episode + 1:4d} | "
f"Avg Reward: {avg_reward:6.2f} | "
f"Avg Length: {avg_length:5.1f} | "
f"Loss: {loss:8.4f}")
print("\n4. VISUALIZING RESULTS...")
agent.visualize_training()
print("\n5. EVALUATING LEARNED POLICY...")
mean_reward, std_reward, eval_rewards = agent.evaluate(env, num_episodes=20)
print(f"\n Evaluation over 20 episodes:")
print(f" Average reward: {mean_reward:.2f} ± {std_reward:.2f}")
print(f" Best episode: {np.max(eval_rewards):.2f}")
print(f" Worst episode: {np.min(eval_rewards):.2f}")
# Success rate (reached goal)
success_rate = sum(1 for r in eval_rewards if r > 0) / len(eval_rewards)
print(f" Success rate: {success_rate:.1%}")
print("\n6. ANALYZING LEARNED POLICY...")
# Visualize policy for all states
print("\n Learned action probabilities:")
states = [np.zeros(5) for _ in range(5)]
for i in range(5):
states[i][i] = 1
with torch.no_grad():
for i, state in enumerate(states):
state_tensor = torch.FloatTensor(state).unsqueeze(0)
logits = agent.policy_net.policy_net(state_tensor)
probs = F.softmax(logits, dim=-1).numpy().flatten()
arrow = '←' if probs[0] > probs[1] else '→'
print(f" State {i}: P(left)={probs[0]:.3f}, P(right)={probs[1]:.3f} {arrow}")
print("\n7. DEMONSTRATION EPISODE...")
print("\n Running one episode with learned policy:")
state = env.reset()
done = False
total_reward = 0
steps = 0
print(" Initial state:")
env.render()
while not done and steps < 20:
action, _, probs = agent.select_action(state, training=False)
state, reward, done = env.step(action)
total_reward += reward
steps += 1
print(f" Step {steps}: Action={'left' if action == 0 else 'right'}, "
f"Reward={reward:.1f}, Total={total_reward:.1f}")
env.render()
print(f"\n Episode finished: Total reward={total_reward:.1f}, Steps={steps}")
print("\n8. KEY INSIGHTS FROM REINFORCE:")
print("\n Strengths:")
print(" 1. Simple to implement")
print(" 2. Direct policy optimization")
print(" 3. Handles stochastic policies naturally")
print("\n Weaknesses:")
print(" 1. High variance (Monte Carlo)")
print(" 2. Sample inefficient")
print(" 3. Can be unstable")
print("\n Improvements (next sections):")
print(" 1. Baseline subtraction (reduce variance)")
print(" 2. Actor-Critic methods")
print(" 3. Trust Region methods")
return agent, env
# Run REINFORCE demo
reinforce_agent, reinforce_env = reinforce_demo()class ActorCriticAgent:
"""Actor-Critic method combining policy and value learning"""
def __init__(self, state_dim, action_dim, hidden_dim=128,
gamma=0.99, lr_actor=0.001, lr_critic=0.001):
"""
Initialize Actor-Critic agent
Args:
state_dim: Dimension of state space
action_dim: Number of possible actions
hidden_dim: Size of hidden layers
gamma: Discount factor
lr_actor: Learning rate for actor
lr_critic: Learning rate for critic
"""
self.state_dim = state_dim
self.action_dim = action_dim
self.gamma = gamma
# Build networks
self.actor = self._build_actor_network(state_dim, hidden_dim, action_dim)
self.critic = self._build_critic_network(state_dim, hidden_dim)
# Optimizers
self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=lr_actor)
self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=lr_critic)
# Training history
self.episode_rewards = []
self.episode_lengths = []
self.actor_losses = []
self.critic_losses = []
def _build_actor_network(self, state_dim, hidden_dim, action_dim):
"""Build actor (policy) network"""
class ActorNetwork(nn.Module):
def __init__(self, state_dim, hidden_dim, action_dim):
super().__init__()
self.net = nn.Sequential(
nn.Linear(state_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, action_dim)
)
def forward(self, x):
return self.net(x)
def get_action(self, state):
"""Sample action and get log probability"""
logits = self.forward(state)
probs = F.softmax(logits, dim=-1)
dist = torch.distributions.Categorical(probs)
action = dist.sample()
log_prob = dist.log_prob(action)
return action.item(), log_prob, probs.detach()
return ActorNetwork(state_dim, hidden_dim, action_dim)
def _build_critic_network(self, state_dim, hidden_dim):
"""Build critic (value) network"""
class CriticNetwork(nn.Module):
def __init__(self, state_dim, hidden_dim):
super().__init__()
self.net = nn.Sequential(
nn.Linear(state_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, 1)
)
def forward(self, x):
return self.net(x)
return CriticNetwork(state_dim, hidden_dim)
def select_action(self, state, training=True):
"""
Select action using actor network
Returns:
action: Selected action
log_prob: Log probability (if training)
value: State value (if training)
probs: Action probabilities
"""
state_tensor = torch.FloatTensor(state).unsqueeze(0)
if training:
action, log_prob, probs = self.actor.get_action(state_tensor)
value = self.critic(state_tensor)
return action, log_prob, value, probs
else:
with torch.no_grad():
logits = self.actor(state_tensor)
probs = F.softmax(logits, dim=-1)
action = torch.argmax(probs).item()
value = self.critic(state_tensor)
return action, None, value, probs
def compute_advantages(self, rewards, values, next_values, dones):
"""
Compute advantages using TD error
Args:
rewards: List of rewards
values: List of value estimates
next_values: List of next state values
dones: List of done flags
Returns:
advantages: Computed advantages
returns: Estimated returns
"""
advantages = []
returns = []
# Compute TD errors
for t in range(len(rewards)):
if dones[t]:
# Terminal state
td_target = rewards[t]
else:
td_target = rewards[t] + self.gamma * next_values[t]
td_error = td_target - values[t]
advantages.append(td_error)
# Also compute returns for critic training
G = 0
for k in range(t, len(rewards)):
G += (self.gamma ** (k - t)) * rewards[k]
returns.append(G)
return advantages, returns
def update(self, log_probs, values, advantages, returns):
"""
Update actor and critic networks
Returns:
actor_loss: Actor loss value
critic_loss: Critic loss value
"""
# Convert to tensors
log_probs = torch.stack(log_probs)
values = torch.cat(values).squeeze()
advantages = torch.FloatTensor(advantages)
returns = torch.FloatTensor(returns)
# Normalize advantages (reduces variance)
advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
# Actor loss: -Σ log_prob * advantage
actor_loss = -(log_probs * advantages).sum()
# Critic loss: MSE between predicted values and returns
critic_loss = F.mse_loss(values, returns)
# Update actor
self.actor_optimizer.zero_grad()
actor_loss.backward(retain_graph=True)
torch.nn.utils.clip_grad_norm_(self.actor.parameters(), 0.5)
self.actor_optimizer.step()
# Update critic
self.critic_optimizer.zero_grad()
critic_loss.backward()
torch.nn.utils.clip_grad_norm_(self.critic.parameters(), 0.5)
self.critic_optimizer.step()
return actor_loss.item(), critic_loss.item()
def train_episode(self, env, max_steps=1000, render=False):
"""
Train for one episode
Returns:
episode_reward: Total reward
episode_length: Episode length
actor_loss: Actor loss
critic_loss: Critic loss
"""
state = env.reset()
done = False
episode_reward = 0
episode_length = 0
# Store episode data
rewards = []
log_probs = []
values = []
next_values = []
dones = []
while not done and episode_length < max_steps:
if render:
env.render()
# Select action
action, log_prob, value, _ = self.select_action(state, training=True)
# Take action
next_state, reward, done = env.step(action)
# Get value of next state
next_state_tensor = torch.FloatTensor(next_state).unsqueeze(0)
with torch.no_grad():
next_value = self.critic(next_state_tensor)
# Store data
rewards.append(reward)
log_probs.append(log_prob)
values.append(value)
next_values.append(next_value)
dones.append(done)
# Update
episode_reward += reward
episode_length += 1
state = next_state
# Compute advantages and update
advantages, returns = self.compute_advantages(rewards,
[v.item() for v in values],
[v.item() for v in next_values],
dones)
actor_loss, critic_loss = self.update(log_probs, values, advantages, returns)
# Store history
self.episode_rewards.append(episode_reward)
self.episode_lengths.append(episode_length)
self.actor_losses.append(actor_loss)
self.critic_losses.append(critic_loss)
return episode_reward, episode_length, actor_loss, critic_loss
def visualize_training(self):
"""Visualize training progress"""
if not self.episode_rewards:
print("No training data to visualize")
return
fig, axes = plt.subplots(2, 3, figsize=(15, 10))
# Plot 1: Learning curve
ax = axes[0, 0]
window = min(50, len(self.episode_rewards) // 10)
if window > 1:
smoothed_rewards = np.convolve(self.episode_rewards,
np.ones(window)/window,
mode='valid')
ax.plot(smoothed_rewards)
ax.set_xlabel('Episode')
ax.set_ylabel(f'Average Reward ({window}-episode window)')
ax.set_title('Learning Curve')
ax.grid(True, alpha=0.3)
# Plot 2: Loss curves
ax = axes[0, 1]
if self.actor_losses and self.critic_losses:
loss_window = max(1, len(self.actor_losses) // 50)
if loss_window > 1:
smoothed_actor = np.convolve(self.actor_losses,
np.ones(loss_window)/loss_window,
mode='valid')
smoothed_critic = np.convolve(self.critic_losses,
np.ones(loss_window)/loss_window,
mode='valid')
ax.plot(smoothed_actor, label='Actor Loss', alpha=0.7)
ax.plot(smoothed_critic, label='Critic Loss', alpha=0.7)
ax.set_xlabel('Episode')
ax.set_ylabel(f'Smoothed Loss (window={loss_window})')
ax.set_title('Actor and Critic Losses')
ax.set_yscale('log')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 3: Value function learning
ax = axes[0, 2]
# This would require tracking value estimates
ax.text(0.5, 0.5, 'Value function analysis\nwould go here',
ha='center', va='center', transform=ax.transAxes, fontsize=12)
ax.set_title('Critic Learning')
ax.axis('off')
# Plot 4: Advantage distribution
ax = axes[1, 0]
# Simplified advantage visualization
if hasattr(self, 'advantages_history'):
if len(self.advantages_history) > 10:
recent_advantages = np.concatenate(self.advantages_history[-10:])
ax.hist(recent_advantages, bins=30, alpha=0.7, edgecolor='black')
ax.set_xlabel('Advantage')
ax.set_ylabel('Frequency')
ax.set_title('Recent Advantage Distribution')
ax.grid(True, alpha=0.3)
# Plot 5: Policy entropy (exploration)
ax = axes[1, 1]
if hasattr(self, 'entropy_history'):
if len(self.entropy_history) > window:
smoothed_entropy = np.convolve(self.entropy_history,
np.ones(window)/window,
mode='valid')
ax.plot(smoothed_entropy)
ax.set_xlabel('Episode')
ax.set_ylabel(f'Policy Entropy ({window}-episode window)')
ax.set_title('Exploration Over Time')
ax.grid(True, alpha=0.3)
# Plot 6: Gradient analysis
ax = axes[1, 2]
if self.actor_losses:
# Compute approximate gradient norms
grad_changes = []
for i in range(1, len(self.actor_losses), 10):
if i < len(self.actor_losses):
grad_changes.append(abs(self.actor_losses[i] - self.actor_losses[i-1]))
if grad_changes:
ax.plot(grad_changes)
ax.set_xlabel('Training Step (every 10th)')
ax.set_ylabel('Loss Change Magnitude')
ax.set_title('Training Stability')
ax.set_yscale('log')
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
def actor_critic_demo():
"""Demonstrate Actor-Critic algorithm"""
print("="*70)
print("ACTOR-CRITIC DEMONSTRATION")
print("="*70)
# Create environment
class ACEnvironment:
"""Environment for Actor-Critic demo"""
def __init__(self, n_states=7):
self.n_states = n_states
self.goal = n_states - 1
self.state = 0
# Actions: 0=left, 1=right
self.actions = [0, 1]
self.action_space = type('ActionSpace', (), {'n': 2})()
# Create a more interesting reward structure
self.reward_map = {
0: -0.1, # Starting point
1: -0.5, # Pit
2: 1.0, # Bonus
3: -0.2, # Neutral
4: -0.8, # Big pit
5: 2.0, # Big bonus
6: 10.0 # Goal
}
def reset(self):
self.state = 0
return self._get_state()
def step(self, action):
if action == 0: # left
self.state = max(0, self.state - 1)
else: # right
self.state = min(self.n_states - 1, self.state + 1)
# Get reward
reward = self.reward_map[self.state]
# Check if done
done = (self.state == self.goal)
return self._get_state(), reward, done
def _get_state(self):
"""One-hot encoded state"""
state = np.zeros(self.n_states)
state[self.state] = 1
return state
def render(self):
"""Visualization"""
line = ['_'] * self.n_states
symbols = {0: 'S', 1: 'P', 2: 'B', 3: '_', 4: 'X', 5: '$', 6: 'G'}
for i in range(self.n_states):
line[i] = symbols.get(i, '_')
# Add agent
if 0 <= self.state < self.n_states:
line[self.state] = 'A'
print('[' + ' '.join(line) + ']')
print(f"State: {self.state}, Reward: {self.reward_map.get(self.state, 0):.1f}")
print("\n1. ENVIRONMENT SETUP:")
print(" Complex chain environment:")
print(" S=Start, P=Pit, B=Bonus, X=Big pit, $=Big bonus, G=Goal")
print(" Rewards: P=-0.5, B=+1.0, X=-0.8, $=+2.0, G=+10.0")
env = ACEnvironment(n_states=7)
print("\n2. AGENT INITIALIZATION:")
print(" Using Actor-Critic method")
print(" Actor network: 7 → 64 → 64 → 2")
print(" Critic network: 7 → 64 → 64 → 1")
print(" Learning rates: Actor=0.001, Critic=0.001")
agent = ActorCriticAgent(
state_dim=7,
action_dim=2,
hidden_dim=64,
gamma=0.99,
lr_actor=0.001,
lr_critic=0.001
)
print("\n3. TRAINING AGENT...")
print(" (Training for 1000 episodes)")
num_episodes = 1000
print_every = 100
for episode in range(num_episodes):
episode_reward, episode_length, actor_loss, critic_loss = agent.train_episode(
env, max_steps=100
)
if (episode + 1) % print_every == 0:
avg_reward = np.mean(agent.episode_rewards[-print_every:])
avg_length = np.mean(agent.episode_lengths[-print_every:])
print(f" Episode {episode + 1:4d} | "
f"Avg Reward: {avg_reward:6.2f} | "
f"Avg Length: {avg_length:5.1f} | "
f"Actor Loss: {actor_loss:8.4f} | "
f"Critic Loss: {critic_loss:8.4f}")
print("\n4. VISUALIZING RESULTS...")
agent.visualize_training()
print("\n5. EVALUATING LEARNED POLICY...")
# Simple evaluation
test_episodes = 20
test_rewards = []
for _ in range(test_episodes):
state = env.reset()
done = False
episode_reward = 0
while not done:
action, _, value, _ = agent.select_action(state, training=False)
state, reward, done = env.step(action)
episode_reward += reward
test_rewards.append(episode_reward)
mean_reward = np.mean(test_rewards)
std_reward = np.std(test_rewards)
print(f"\n Evaluation over {test_episodes} episodes:")
print(f" Average reward: {mean_reward:.2f} ± {std_reward:.2f}")
print(f" Best episode: {np.max(test_rewards):.2f}")
print(f" Worst episode: {np.min(test_rewards):.2f}")
print("\n6. ANALYZING LEARNED VALUE FUNCTION...")
# Visualize learned values
states = [np.zeros(7) for _ in range(7)]
for i in range(7):
states[i][i] = 1
with torch.no_grad():
print("\n State values (learned by critic):")
for i, state in enumerate(states):
state_tensor = torch.FloatTensor(state).unsqueeze(0)
value = agent.critic(state_tensor).item()
print(f" State {i}: V(s) = {value:.3f}")
print("\n7. ANALYZING LEARNED POLICY...")
print("\n Action probabilities and policy:")
with torch.no_grad():
for i, state in enumerate(states):
state_tensor = torch.FloatTensor(state).unsqueeze(0)
logits = agent.actor(state_tensor)
probs = F.softmax(logits, dim=-1).numpy().flatten()
# Determine optimal action
if probs[0] > probs[1]:
action = "left"
confidence = probs[0]
else:
action = "right"
confidence = probs[1]
symbol = {0: 'S', 1: 'P', 2: 'B', 3: '_', 4: 'X', 5: '$', 6: 'G'}[i]
print(f" {symbol} (State {i}): P(left)={probs[0]:.3f}, "
f"P(right)={probs[1]:.3f} → {action} ({confidence:.1%} confident)")
print("\n8. DEMONSTRATION EPISODE...")
print("\n Running one episode with learned policy:")
state = env.reset()
done = False
total_reward = 0
steps = 0
print("\n Initial state:")
env.render()
while not done and steps < 15:
action, _, value, probs = agent.select_action(state, training=False)
state, reward, done = env.step(action)
total_reward += reward
steps += 1
print(f"\n Step {steps}:")
print(f" Action: {'left' if action == 0 else 'right'}")
print(f" P(left)={probs[0][0].item():.3f}, P(right)={probs[0][1].item():.3f}")
print(f" Value estimate: {value.item():.3f}")
print(f" Reward: {reward:.1f}, Total: {total_reward:.1f}")
env.render()
print(f"\n Episode finished: Total reward={total_reward:.1f}, Steps={steps}")
print("\n9. COMPARISON WITH REINFORCE:")
print("\n Actor-Critic advantages:")
print(" 1. Lower variance (uses value function as baseline)")
print(" 2. Online updates (no need to wait for episode end)")
print(" 3. More stable learning")
print("\n Implementation details:")
print(" 1. Actor updates policy using advantages")
print(" 2. Critic learns value function")
print(" 3. Advantage = TD error = r + γV(s') - V(s)")
return agent, env
# Run Actor-Critic demo
ac_agent, ac_env = actor_critic_demo()class AdvantageEstimation:
"""Advanced advantage estimation methods"""
def __init__(self):
self.explain_advantages()
def explain_advantages(self):
"""Explain different advantage estimation methods"""
print("="*70)
print("ADVANTAGE ESTIMATION METHODS")
print("="*70)
print("\nTHE PROBLEM WITH MONTE CARLO:")
print("REINFORCE uses: ∇_θ J(θ) = E[Σ_t ∇_θ log π_θ(a_t|s_t) G_t]")
print("Where G_t = Σ_{k=t}^T γ^{k-t} r_k")
print("Problem: High variance (full Monte Carlo returns)")
print("\n" + "-"*70)
print("SOLUTION: USE BASELINES")
print("-"*70)
print("\nWith baseline b(s_t):")
print("∇_θ J(θ) = E[Σ_t ∇_θ log π_θ(a_t|s_t) (G_t - b(s_t))]")
print("\nGood baseline choices:")
print("1. State value V(s_t) ← What Actor-Critic uses!")
print("2. Average return")
print("3. Learned baseline network")
print("\n" + "-"*70)
print("TD ADVANTAGE (ACTOR-CRITIC):")
print("-"*70)
print("\nTD(0) Advantage:")
print("A(s_t, a_t) = r_t + γV(s_{t+1}) - V(s_t)")
print("Pros: Low variance")
print("Cons: High bias (1-step)")
print("\n" + "-"*70)
print("N-STEP ADVANTAGE:")
print("-"*70)
print("\nN-step Advantage:")
print("A^n(s_t, a_t) = Σ_{k=0}^{n-1} γ^k r_{t+k} + γ^n V(s_{t+n}) - V(s_t)")
print("Tradeoff: Between TD (bias) and MC (variance)")
def generalized_advantage_estimation(self, rewards, values, next_values,
dones, gamma=0.99, lambda_param=0.95):
"""
Generalized Advantage Estimation (GAE)
Args:
rewards: List of rewards
values: List of value estimates V(s_t)
next_values: List of next values V(s_{t+1})
dones: List of done flags
gamma: Discount factor
lambda_param: GAE parameter (0=TD, 1=MC)
Returns:
advantages: GAE advantages
returns: Estimated returns
"""
advantages = []
returns = []
# Compute TD errors
td_errors = []
for t in range(len(rewards)):
if dones[t]:
td_target = rewards[t]
else:
td_target = rewards[t] + gamma * next_values[t]
td_error = td_target - values[t]
td_errors.append(td_error)
# Compute GAE advantages
advantage = 0
for t in reversed(range(len(rewards))):
if dones[t]:
advantage = 0
advantage = td_errors[t] + gamma * lambda_param * advantage
advantages.insert(0, advantage)
# Compute returns for critic training
for t in range(len(rewards)):
G = 0
for k in range(t, len(rewards)):
G += (gamma ** (k - t)) * rewards[k]
returns.append(G)
return advantages, returns
def compare_advantage_methods(self):
"""Compare different advantage estimation methods"""
print("\n" + "="*70)
print("COMPARING ADVANTAGE ESTIMATION METHODS")
print("="*70)
# Generate sample trajectory
np.random.seed(42)
T = 20 # Episode length
rewards = np.random.normal(0, 1, T)
values = np.cumsum(np.random.normal(0, 0.1, T)) # Simulated value estimates
next_values = np.concatenate([values[1:], [0]]) # Next state values
dones = [False] * (T-1) + [True] # Last state is terminal
gamma = 0.9
lambda_param = 0.95
# Compute different advantages
# Monte Carlo advantages
mc_advantages = []
for t in range(T):
G = 0
for k in range(t, T):
G += (gamma ** (k - t)) * rewards[k]
mc_advantages.append(G - values[t])
# TD(0) advantages
td_advantages = []
for t in range(T):
if dones[t]:
td_target = rewards[t]
else:
td_target = rewards[t] + gamma * next_values[t]
td_advantages.append(td_target - values[t])
# GAE advantages
gae_advantages, _ = self.generalized_advantage_estimation(
rewards, values, next_values, dones, gamma, lambda_param
)
# Plot comparison
fig, axes = plt.subplots(2, 2, figsize=(12, 10))
# Plot 1: Advantages over time
ax = axes[0, 0]
time = range(T)
ax.plot(time, mc_advantages, 'b-', label='Monte Carlo', linewidth=2, alpha=0.7)
ax.plot(time, td_advantages, 'r-', label='TD(0)', linewidth=2, alpha=0.7)
ax.plot(time, gae_advantages, 'g-', label=f'GAE(λ={lambda_param})', linewidth=2, alpha=0.7)
ax.set_xlabel('Time step t')
ax.set_ylabel('Advantage A(s_t, a_t)')
ax.set_title('Advantage Estimates Comparison')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 2: Variance comparison
ax = axes[0, 1]
methods = ['Monte Carlo', 'TD(0)', 'GAE']
variances = [np.var(mc_advantages), np.var(td_advantages), np.var(gae_advantages)]
bars = ax.bar(methods, variances, color=['blue', 'red', 'green'], alpha=0.7)
ax.set_ylabel('Variance of Advantages')
ax.set_title('Variance Comparison')
ax.set_yscale('log')
# Add value labels
for bar, var in zip(bars, variances):
ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() * 1.1,
f'{var:.3f}', ha='center', va='bottom')
# Plot 3: Bias-Variance tradeoff
ax = axes[1, 0]
# Simulate different lambda values
lambda_values = np.linspace(0, 1, 11)
variances = []
for lam in lambda_values:
advantages, _ = self.generalized_advantage_estimation(
rewards, values, next_values, dones, gamma, lam
)
variances.append(np.var(advantages))
ax.plot(lambda_values, variances, 'b-o', linewidth=2, markersize=6)
ax.set_xlabel('λ parameter')
ax.set_ylabel('Variance of Advantages')
ax.set_title('Bias-Variance Tradeoff (λ=0:TD, λ=1:MC)')
ax.grid(True, alpha=0.3)
# Mark special points
ax.scatter([0, 1], [variances[0], variances[-1]],
color=['red', 'green'], s=100, zorder=5)
ax.text(0.05, variances[0], 'TD(0)\n(low variance,\nhigh bias)',
ha='left', va='bottom')
ax.text(0.95, variances[-1], 'Monte Carlo\n(high variance,\nlow bias)',
ha='right', va='top')
# Plot 4: Mathematical formulas
ax = axes[1, 1]
ax.axis('off')
formulas = [
"Monte Carlo Advantage:",
"A^{MC}(s_t) = G_t - V(s_t)",
"",
"TD(0) Advantage:",
"A^{TD}(s_t) = r_t + γV(s_{t+1}) - V(s_t)",
"",
"N-step Advantage:",
"A^{n}(s_t) = Σ_{k=0}^{n-1} γ^k r_{t+k}",
" + γ^n V(s_{t+n}) - V(s_t)",
"",
"GAE Advantage:",
"A^{GAE}(s_t) = Σ_{l=0}^{∞} (γλ)^l δ_{t+l}",
"where δ_t = r_t + γV(s_{t+1}) - V(s_t)"
]
y_pos = 0.95
for formula in formulas:
if '=' in formula or 'where' in formula:
fontsize = 10
else:
fontsize = 11
y_pos -= 0.05
ax.text(0.05, y_pos, formula, fontsize=fontsize,
verticalalignment='top', transform=ax.transAxes)
y_pos -= 0.08
ax.set_title('Mathematical Formulas')
plt.tight_layout()
plt.show()
print("\nKEY INSIGHTS:")
print("\n1. Monte Carlo (λ=1):")
print(" - Unbiased estimator")
print(" - High variance")
print(" - Uses full returns")
print("\n2. TD(0) (λ=0):")
print(" - Biased estimator")
print(" - Low variance")
print(" - Uses 1-step returns")
print("\n3. GAE (0<λ<1):")
print(" - Smooth interpolation between TD and MC")
print(" - Adjustable bias-variance tradeoff")
print(" - Typically λ=0.95 works well")
print("\nPRACTICAL RECOMMENDATIONS:")
print("1. Start with λ=0.95 for most problems")
print("2. Use GAE for stable learning")
print("3. Monitor advantage variance during training")
print("4. Adjust λ based on environment stochasticity")
return mc_advantages, td_advantages, gae_advantages
# Run advantage estimation comparison
mc_adv, td_adv, gae_adv = AdvantageEstimation().compare_advantage_methods()def lunar_lander_project():
"""Lunar Lander with Policy Gradient methods"""
print("="*70)
print("LUNAR LANDER PROJECT")
print("="*70)
try:
import gymnasium as gym
gym_available = True
except ImportError:
print("gymnasium not available. Creating custom environment...")
gym_available = False
if gym_available:
# Use actual LunarLander environment
print("\n1. INITIALIZING LUNAR LANDER ENVIRONMENT...")
env = gym.make('LunarLander-v2', render_mode='rgb_array')
print(f" State dimension: {env.observation_space.shape[0]}")
print(f" Action space: {env.action_space.n} discrete actions")
print("\n Actions: 0=do nothing, 1=left engine, 2=main engine, 3=right engine")
print(" Goal: Land softly between flags")
print(" Reward: 100-140 for landing, -100 for crashing, -0.3/frame for main engine")
# Create a wrapper for our training loop
class GymEnvWrapper:
def __init__(self, gym_env):
self.env = gym_env
self.state_dim = gym_env.observation_space.shape[0]
self.action_dim = gym_env.action_space.n
def reset(self):
state, _ = self.env.reset()
return state
def step(self, action):
state, reward, terminated, truncated, _ = self.env.step(action)
done = terminated or truncated
return state, reward, done
def render(self):
return self.env.render()
wrapped_env = GymEnvWrapper(env)
state_dim = wrapped_env.state_dim
action_dim = wrapped_env.action_dim
else:
# Create a simplified Lunar Lander environment
print("\n1. CREATING SIMPLIFIED LUNAR LANDER ENVIRONMENT...")
class SimpleLunarLander:
def __init__(self):
# State: [x, y, vx, vy, angle, angular_velocity, left_leg, right_leg]
self.state_dim = 8
self.action_dim = 4 # 0: none, 1: left, 2: main, 3: right
# Initial state
self.reset()
def reset(self):
# Start at random position
self.x = np.random.uniform(-0.5, 0.5)
self.y = np.random.uniform(1.0, 1.5)
self.vx = np.random.uniform(-0.2, 0.2)
self.vy = np.random.uniform(-0.5, 0)
self.angle = np.random.uniform(-0.2, 0.2)
self.angular_velocity = np.random.uniform(-0.1, 0.1)
self.left_leg = 0
self.right_leg = 0
self.steps = 0
return self._get_state()
def step(self, action):
# Simplified physics
dt = 0.05
g = -1.5 # Gravity
# Apply action
if action == 1: # left engine
self.vx += 0.1 * dt
self.vy += 0.5 * dt
self.angular_velocity += 0.1 * dt
elif action == 2: # main engine
self.vy += 1.0 * dt
elif action == 3: # right engine
self.vx -= 0.1 * dt
self.vy += 0.5 * dt
self.angular_velocity -= 0.1 * dt
# Update physics
self.x += self.vx * dt
self.y += self.vy * dt + 0.5 * g * dt**2
self.vy += g * dt
self.angle += self.angular_velocity * dt
# Keep angle in reasonable range
self.angle = np.clip(self.angle, -np.pi/2, np.pi/2)
# Update leg contact
if self.y <= 0.1 and abs(self.angle) < 0.2:
if self.x < -0.1:
self.left_leg = 1
elif self.x > 0.1:
self.right_leg = 1
# Calculate reward
reward = self._calculate_reward(action)
# Check if done
self.steps += 1
done = (self.y <= 0) or (self.steps >= 200)
return self._get_state(), reward, done
def _calculate_reward(self, action):
# Base reward for being alive
reward = 0.1
# Penalty for using main engine
if action == 2:
reward -= 0.3
# Reward for being upright
reward += 0.1 * (1.0 - abs(self.angle) / (np.pi/2))
# Reward for being near center
reward += 0.1 * (1.0 - abs(self.x) / 0.5)
# Reward for low speed
speed = np.sqrt(self.vx**2 + self.vy**2)
reward += 0.1 * (1.0 - min(speed, 2.0) / 2.0)
# Landing bonus
if self.y <= 0:
if abs(self.angle) < 0.2 and abs(self.x) < 0.1 and speed < 0.5:
reward += 100 # Successful landing
if self.left_leg and self.right_leg:
reward += 40 # Both legs touched
else:
reward -= 100 # Crash
return reward
def _get_state(self):
return np.array([
self.x, self.y, self.vx, self.vy,
self.angle, self.angular_velocity,
self.left_leg, self.right_leg
])
def render(self):
"""Simple text render"""
print(f"Pos: ({self.x:.2f}, {self.y:.2f}), "
f"Vel: ({self.vx:.2f}, {self.vy:.2f}), "
f"Angle: {np.degrees(self.angle):.1f}°, "
f"Steps: {self.steps}")
wrapped_env = SimpleLunarLander()
state_dim = wrapped_env.state_dim
action_dim = wrapped_env.action_dim
print("\n2. IMPLEMENTING ADVANCED POLICY GRADIENT AGENT...")
class AdvancedPGAgent:
"""Advanced Policy Gradient agent with GAE and entropy regularization"""
def __init__(self, state_dim, action_dim, hidden_dim=256,
gamma=0.99, gae_lambda=0.95, lr=3e-4,
entropy_coef=0.01, value_coef=0.5, max_grad_norm=0.5):
"""
Initialize advanced PG agent
Args:
state_dim: Dimension of state space
action_dim: Number of possible actions
hidden_dim: Size of hidden layers
gamma: Discount factor
gae_lambda: GAE parameter
lr: Learning rate
entropy_coef: Entropy regularization coefficient
value_coef: Value loss coefficient
max_grad_norm: Gradient clipping norm
"""
self.state_dim = state_dim
self.action_dim = action_dim
self.gamma = gamma
self.gae_lambda = gae_lambda
self.entropy_coef = entropy_coef
self.value_coef = value_coef
self.max_grad_norm = max_grad_norm
# Build networks
self.actor = self._build_actor(state_dim, hidden_dim, action_dim)
self.critic = self._build_critic(state_dim, hidden_dim)
# Single optimizer for both networks
parameters = list(self.actor.parameters()) + list(self.critic.parameters())
self.optimizer = optim.Adam(parameters, lr=lr)
# Training history
self.episode_rewards = []
self.episode_lengths = []
self.losses = []
self.entropies = []
self.value_losses = []
self.policy_losses = []
# For GAE computation
self.states = []
self.actions = []
self.rewards = []
self.values = []
self.next_values = []
self.dones = []
self.log_probs = []
def _build_actor(self, state_dim, hidden_dim, action_dim):
"""Build actor network"""
return nn.Sequential(
nn.Linear(state_dim, hidden_dim),
nn.Tanh(),
nn.Linear(hidden_dim, hidden_dim),
nn.Tanh(),
nn.Linear(hidden_dim, action_dim)
)
def _build_critic(self, state_dim, hidden_dim):
"""Build critic network"""
return nn.Sequential(
nn.Linear(state_dim, hidden_dim),
nn.Tanh(),
nn.Linear(hidden_dim, hidden_dim),
nn.Tanh(),
nn.Linear(hidden_dim, 1)
)
def select_action(self, state, training=True):
"""
Select action using current policy
Returns:
action: Selected action
log_prob: Log probability
value: State value
entropy: Policy entropy
"""
state_tensor = torch.FloatTensor(state).unsqueeze(0)
# Get action probabilities
logits = self.actor(state_tensor)
probs = F.softmax(logits, dim=-1)
dist = torch.distributions.Categorical(probs)
# Sample action
if training:
action = dist.sample()
else:
action = torch.argmax(probs)
log_prob = dist.log_prob(action)
entropy = dist.entropy()
value = self.critic(state_tensor)
return action.item(), log_prob, value, entropy.item()
def store_transition(self, state, action, reward, next_state,
log_prob, value, next_value, done):
"""Store transition for GAE computation"""
self.states.append(state)
self.actions.append(action)
self.rewards.append(reward)
self.values.append(value.item())
self.next_values.append(next_value.item())
self.dones.append(done)
self.log_probs.append(log_prob)
def compute_gae(self):
"""Compute Generalized Advantage Estimation"""
advantages = []
returns = []
# Compute TD errors
td_errors = []
for t in range(len(self.rewards)):
if self.dones[t]:
td_target = self.rewards[t]
else:
td_target = self.rewards[t] + self.gamma * self.next_values[t]
td_error = td_target - self.values[t]
td_errors.append(td_error)
# Compute GAE advantages
advantage = 0
for t in reversed(range(len(self.rewards))):
if self.dones[t]:
advantage = 0
advantage = td_errors[t] + self.gamma * self.gae_lambda * advantage
advantages.insert(0, advantage)
# Compute returns
for t in range(len(self.rewards)):
G = 0
for k in range(t, len(self.rewards)):
G += (self.gamma ** (k - t)) * self.rewards[k]
returns.append(G)
return advantages, returns
def update(self):
"""Update networks using stored transitions"""
if not self.states:
return 0, 0, 0
# Compute advantages and returns
advantages, returns = self.compute_gae()
# Convert to tensors
states_tensor = torch.FloatTensor(np.array(self.states))
actions_tensor = torch.LongTensor(self.actions)
log_probs_tensor = torch.stack(self.log_probs)
advantages_tensor = torch.FloatTensor(advantages)
returns_tensor = torch.FloatTensor(returns)
# Normalize advantages
advantages_tensor = (advantages_tensor - advantages_tensor.mean()) / (advantages_tensor.std() + 1e-8)
# Get current policy distribution
current_logits = self.actor(states_tensor)
current_probs = F.softmax(current_logits, dim=-1)
current_dist = torch.distributions.Categorical(current_probs)
# Get current log probabilities
current_log_probs = current_dist.log_prob(actions_tensor)
# Policy loss
ratio = torch.exp(current_log_probs - log_probs_tensor.detach())
policy_loss = -(ratio * advantages_tensor).mean()
# Value loss
values = self.critic(states_tensor).squeeze()
value_loss = F.mse_loss(values, returns_tensor)
# Entropy bonus
entropy = current_dist.entropy().mean()
# Total loss
loss = (policy_loss
+ self.value_coef * value_loss
- self.entropy_coef * entropy)
# Optimize
self.optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(self.actor.parameters(), self.max_grad_norm)
torch.nn.utils.clip_grad_norm_(self.critic.parameters(), self.max_grad_norm)
self.optimizer.step()
# Clear buffers
self.clear_buffers()
return loss.item(), policy_loss.item(), value_loss.item(), entropy.item()
def clear_buffers(self):
"""Clear stored transitions"""
self.states.clear()
self.actions.clear()
self.rewards.clear()
self.values.clear()
self.next_values.clear()
self.dones.clear()
self.log_probs.clear()
def train_episode(self, env, max_steps=1000, render=False):
"""
Train for one episode
Returns:
episode_reward: Total reward
episode_length: Episode length
loss: Training loss
"""
state = env.reset()
done = False
episode_reward = 0
episode_length = 0
while not done and episode_length < max_steps:
if render:
env.render()
# Select action
action, log_prob, value, entropy = self.select_action(state, training=True)
# Take action
next_state, reward, done = env.step(action)
# Get next state value
next_state_tensor = torch.FloatTensor(next_state).unsqueeze(0)
with torch.no_grad():
next_value = self.critic(next_state_tensor)
# Store transition
self.store_transition(state, action, reward, next_state,
log_prob, value, next_value, done)
# Update
episode_reward += reward
episode_length += 1
state = next_state
# Update policy
loss, policy_loss, value_loss, entropy_val = self.update()
# Store history
self.episode_rewards.append(episode_reward)
self.episode_lengths.append(episode_length)
self.losses.append(loss)
self.policy_losses.append(policy_loss)
self.value_losses.append(value_loss)
self.entropies.append(entropy_val)
return episode_reward, episode_length, loss
def evaluate(self, env, num_episodes=5, render=False):
"""
Evaluate agent performance
Returns:
mean_reward: Average reward
std_reward: Standard deviation
all_rewards: List of rewards
"""
rewards = []
for episode in range(num_episodes):
state = env.reset()
done = False
episode_reward = 0
if render:
print(f"\nEvaluation Episode {episode + 1}:")
while not done:
action, _, _, _ = self.select_action(state, training=False)
state, reward, done = env.step(action)
episode_reward += reward
if render:
env.render()
rewards.append(episode_reward)
if render:
print(f" Episode reward: {episode_reward:.1f}")
mean_reward = np.mean(rewards)
std_reward = np.std(rewards)
return mean_reward, std_reward, rewards
print("\n3. INITIALIZING ADVANCED PG AGENT...")
print(f" State dimension: {state_dim}")
print(f" Action dimension: {action_dim}")
print(f" Network: {state_dim} → 256 → 256 → {action_dim}")
print(f" GAE λ: 0.95, Entropy coefficient: 0.01")
print(f" Learning rate: 3e-4, Max grad norm: 0.5")
agent = AdvancedPGAgent(
state_dim=state_dim,
action_dim=action_dim,
hidden_dim=256,
gamma=0.99,
gae_lambda=0.95,
lr=3e-4,
entropy_coef=0.01,
value_coef=0.5,
max_grad_norm=0.5
)
print("\n4. TRAINING AGENT...")
print(" (Training for 2000 episodes)")
num_episodes = 2000
print_every = 100
best_reward = -float('inf')
for episode in range(num_episodes):
episode_reward, episode_length, loss = agent.train_episode(
wrapped_env, max_steps=500
)
# Track best reward
if episode_reward > best_reward:
best_reward = episode_reward
# Print progress
if (episode + 1) % print_every == 0:
avg_reward = np.mean(agent.episode_rewards[-print_every:])
avg_length = np.mean(agent.episode_lengths[-print_every:])
avg_loss = np.mean(agent.losses[-print_every:])
print(f" Episode {episode + 1:4d} | "
f"Avg Reward: {avg_reward:7.1f} | "
f"Avg Length: {avg_length:5.1f} | "
f"Loss: {avg_loss:8.4f} | "
f"Best: {best_reward:7.1f}")
print("\n5. VISUALIZING RESULTS...")
# Create comprehensive visualization
fig, axes = plt.subplots(3, 3, figsize=(15, 12))
# Plot 1: Learning curve
ax = axes[0, 0]
window = 50
if len(agent.episode_rewards) > window:
smoothed_rewards = np.convolve(agent.episode_rewards,
np.ones(window)/window,
mode='valid')
ax.plot(smoothed_rewards, linewidth=2)
ax.set_xlabel('Episode')
ax.set_ylabel(f'Average Reward ({window}-episode window)')
ax.set_title('Learning Curve')
ax.grid(True, alpha=0.3)
# Mark success threshold (typically > 200 for LunarLander)
ax.axhline(y=200, color='green', linestyle='--', alpha=0.5,
label='Success threshold (~200)')
ax.legend()
# Plot 2: Episode lengths
ax = axes[0, 1]
if len(agent.episode_lengths) > window:
smoothed_lengths = np.convolve(agent.episode_lengths,
np.ones(window)/window,
mode='valid')
ax.plot(smoothed_lengths, linewidth=2)
ax.set_xlabel('Episode')
ax.set_ylabel(f'Average Length ({window}-episode window)')
ax.set_title('Episode Lengths')
ax.grid(True, alpha=0.3)
# Plot 3: Loss components
ax = axes[0, 2]
if agent.losses and agent.policy_losses and agent.value_losses:
loss_window = max(1, len(agent.losses) // 50)
if loss_window > 1:
smoothed_total = np.convolve(agent.losses,
np.ones(loss_window)/loss_window,
mode='valid')
smoothed_policy = np.convolve(agent.policy_losses,
np.ones(loss_window)/loss_window,
mode='valid')
smoothed_value = np.convolve(agent.value_losses,
np.ones(loss_window)/loss_window,
mode='valid')
ax.plot(smoothed_total, label='Total Loss', linewidth=2)
ax.plot(smoothed_policy, label='Policy Loss', linewidth=2, alpha=0.7)
ax.plot(smoothed_value, label='Value Loss', linewidth=2, alpha=0.7)
ax.set_xlabel('Episode')
ax.set_ylabel(f'Smoothed Loss (window={loss_window})')
ax.set_title('Loss Components')
ax.set_yscale('log')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 4: Entropy over time
ax = axes[1, 0]
if agent.entropies:
entropy_window = max(1, len(agent.entropies) // 20)
if entropy_window > 1:
smoothed_entropy = np.convolve(agent.entropies,
np.ones(entropy_window)/entropy_window,
mode='valid')
ax.plot(smoothed_entropy, linewidth=2, color='purple')
ax.set_xlabel('Episode')
ax.set_ylabel(f'Policy Entropy (window={entropy_window})')
ax.set_title('Exploration Over Time')
ax.grid(True, alpha=0.3)
# Theoretical maximum entropy for discrete actions
max_entropy = np.log(action_dim)
ax.axhline(y=max_entropy, color='red', linestyle='--', alpha=0.5,
label=f'Max entropy: {max_entropy:.2f}')
ax.legend()
# Plot 5: Reward distribution
ax = axes[1, 1]
if len(agent.episode_rewards) > 100:
recent_rewards = agent.episode_rewards[-100:]
ax.hist(recent_rewards, bins=20, alpha=0.7, edgecolor='black')
ax.set_xlabel('Episode Reward')
ax.set_ylabel('Frequency')
ax.set_title('Recent Reward Distribution')
ax.grid(True, alpha=0.3)
# Add mean line
mean_reward = np.mean(recent_rewards)
ax.axvline(x=mean_reward, color='red', linestyle='--', alpha=0.7,
label=f'Mean: {mean_reward:.1f}')
ax.legend()
# Plot 6: Learning stages
ax = axes[1, 2]
# Divide training into stages
if len(agent.episode_rewards) > 300:
stage_size = len(agent.episode_rewards) // 3
stages = ['Early', 'Middle', 'Late']
stage_rewards = []
for i in range(3):
start = i * stage_size
end = (i + 1) * stage_size if i < 2 else len(agent.episode_rewards)
stage_rewards.append(agent.episode_rewards[start:end])
# Create box plot
bp = ax.boxplot(stage_rewards, labels=stages, patch_artist=True)
# Color boxes
colors = ['lightblue', 'lightgreen', 'lightcoral']
for patch, color in zip(bp['boxes'], colors):
patch.set_facecolor(color)
ax.set_ylabel('Episode Reward')
ax.set_title('Learning Progress by Stage')
ax.grid(True, alpha=0.3)
# Plot 7: Action distribution (last episode)
ax = axes[2, 0]
# This would require tracking actions during evaluation
ax.text(0.5, 0.5, 'Action distribution analysis\nwould require tracking actions',
ha='center', va='center', transform=ax.transAxes, fontsize=10)
ax.set_title('Action Distribution')
ax.axis('off')
# Plot 8: Value function learning
ax = axes[2, 1]
# Simplified value function analysis
if hasattr(agent, 'value_history'):
ax.text(0.5, 0.5, 'Value function learning curve\nwould be shown here',
ha='center', va='center', transform=ax.transAxes, fontsize=10)
else:
ax.text(0.5, 0.5, 'Value function metrics\nnot tracked in this demo',
ha='center', va='center', transform=ax.transAxes, fontsize=10)
ax.set_title('Critic Learning')
ax.axis('off')
# Plot 9: Success rate over time
ax = axes[2, 2]
if len(agent.episode_rewards) > window:
# Define success as reward > 200 (for LunarLander)
success_threshold = 200
success_rates = []
for i in range(0, len(agent.episode_rewards) - window + 1, window):
window_rewards = agent.episode_rewards[i:i + window]
success_rate = sum(1 for r in window_rewards if r > success_threshold) / len(window_rewards)
success_rates.append(success_rate)
if success_rates:
ax.plot(range(0, len(agent.episode_rewards), window)[:len(success_rates)],
success_rates, 'o-', linewidth=2, markersize=6)
ax.set_xlabel('Episode')
ax.set_ylabel('Success Rate')
ax.set_title(f'Success Rate (reward > {success_threshold})')
ax.set_ylim(-0.05, 1.05)
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\n6. EVALUATING LEARNED AGENT...")
# Run evaluation
eval_episodes = 10
print(f"\n Running evaluation for {eval_episodes} episodes...")
mean_reward, std_reward, eval_rewards = agent.evaluate(
wrapped_env, num_episodes=eval_episodes, render=False
)
print(f"\n Evaluation Results:")
print(f" Average reward: {mean_reward:.1f} ± {std_reward:.1f}")
print(f" Best episode: {np.max(eval_rewards):.1f}")
print(f" Worst episode: {np.min(eval_rewards):.1f}")
# Success rate
success_threshold = 200
success_count = sum(1 for r in eval_rewards if r > success_threshold)
print(f" Success rate (reward > {success_threshold}): {success_count}/{eval_episodes} ({success_count/eval_episodes:.1%})")
print("\n7. KEY INSIGHTS FROM LUNAR LANDER PROJECT:")
print("\n What Policy Gradients learned:")
print(" 1. Balance thrusters to control descent")
print(" 2. Soft landing requires careful timing")
print(" 3. Need to manage fuel (main engine penalty)")
print("\n Challenges observed:")
print(" 1. High-dimensional continuous state space")
print(" 2. Sparse rewards (only at landing)")
print(" 3. Need for exploration in early training")
print("\n Advanced techniques used:")
print(" 1. Generalized Advantage Estimation (GAE)")
print(" 2. Entropy regularization for exploration")
print(" 3. Value function as baseline (Actor-Critic)")
print(" 4. Gradient clipping for stability")
print("\n Comparison with value-based methods:")
print(" 1. Policy gradients handle continuous actions naturally")
print(" 2. Can learn stochastic policies")
print(" 3. Direct optimization of what we care about")
print(" 4. Often more sample-efficient for complex tasks")
print("\n8. NEXT STEPS:")
print(" For even better performance:")
print(" 1. Use PPO (Proximal Policy Optimization)")
print(" 2. Try TRPO (Trust Region Policy Optimization)")
print(" 3. Implement parallel environments")
print(" 4. Add reward shaping")
return agent, wrapped_env
# Run Lunar Lander project
lunar_agent, lunar_env = lunar_lander_project()Chapter 9 Key Takeaways:
-
Policy Gradient Philosophy:
- Directly optimize policy parameters θ
- Output action probabilities π_θ(a|s)
- Natural for continuous/stochastic actions
-
Policy Gradient Theorem:
- ∇_θ J(θ) = E[Σ_t ∇_θ log π_θ(a_t|s_t) G_t]
- Only need log probabilities of actions taken
- Environment dynamics don't matter (model-free)
-
REINFORCE Algorithm:
- Monte Carlo policy gradient
- Simple but high variance
- Collect full episode, then update
- Baseline subtraction reduces variance
-
Actor-Critic Methods:
- Actor: Learns policy π_θ(a|s)
- Critic: Learns value function V_φ(s)
- Advantages: A(s,a) = r + γV(s') - V(s)
- Lower variance than REINFORCE
-
Generalized Advantage Estimation (GAE):
- A^{GAE} = Σ_{l=0}^{∞} (γλ)^l δ_{t+l}
- λ=0: TD(0) (low variance, high bias)
- λ=1: Monte Carlo (high variance, low bias)
- 0<λ<1: Interpolation (adjustable tradeoff)
-
Advanced Techniques:
- Entropy regularization: Encourages exploration
- Gradient clipping: Prevents large updates
- Value function clipping (PPO)
- Trust regions (TRPO)
-
Lunar Lander Project:
- Demonstrated policy gradients on complex task
- Used GAE for stable learning
- Entropy regularization for exploration
- Showed practical implementation details
-
Comparison with Value-Based Methods:
- Policy Gradients: Direct, stochastic, continuous actions
- Value-Based: Indirect, often deterministic, discrete actions
- Actor-Critic: Best of both worlds
-
Practical Implementation Tips:
- Start with REINFORCE for understanding
- Use Actor-Critic for better performance
- Implement GAE with λ=0.95
- Add entropy regularization
- Monitor learning curves closely
- Tune learning rates carefully
I'll provide you with the complete, continuous Chapter 10. Let me write it as a single, cohesive chapter:
# **CHAPTER 10: The Frontier & Putting It All Together**
#### **10.1 Beyond the Basics: Modern RL Algorithms**
```python
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import matplotlib.pyplot as plt
from collections import defaultdict, deque
import random
from IPython.display import display, HTML
class RLFrontier:
"""Exploring the cutting edge of Reinforcement Learning"""
def __init__(self):
self.setup_visualization()
def setup_visualization(self):
"""Setup visualization for the frontier"""
print("="*80)
print("CHAPTER 10: THE REINFORCEMENT LEARNING FRONTIER")
print("="*80)
print("\nWelcome to the final chapter! Here we explore:")
print("• Modern RL algorithms that power today's AI systems")
print("• Real-world challenges and practical solutions")
print("• Multi-agent systems and collaborative AI")
print("• Meta-learning: teaching AI to learn faster")
print("• RLHF: How ChatGPT and similar models are trained")
print("• The future of RL and your role in it")
self.timeline_data = self.create_modern_rl_timeline()
def create_modern_rl_timeline(self):
"""Create timeline of modern RL developments"""
timeline = [
(2015, "DQN (Nature)", "Deep Q-Networks achieve human-level Atari play"),
(2016, "A3C", "Asynchronous Advantage Actor-Critic"),
(2017, "PPO", "Proximal Policy Optimization (simpler than TRPO)"),
(2017, "Rainbow DQN", "Combines 7 DQN improvements"),
(2018, "Soft Actor-Critic (SAC)", "Maximum entropy RL for robotics"),
(2018, "IMPALA", "Massively parallel RL"),
(2019, "MuZero", "Model-based + Model-free, learns dynamics"),
(2020, "Dreamer", "World models for sample efficiency"),
(2021, "Decision Transformer", "RL as sequence modeling"),
(2022, "ChatGPT/RLHF", "Reinforcement Learning from Human Feedback"),
(2023, "AlphaDev", "Discovering faster sorting algorithms"),
(2024, "Sora/Gemini", "Video generation and multimodal RL")
]
print("\nMODERN RL TIMELINE:")
print("-"*80)
for year, name, description in timeline:
print(f"{year}: {name:20s} - {description}")
return timeline
def visualize_algorithm_landscape(self):
"""Visualize the RL algorithm landscape"""
algorithms = {
"Value-Based": ["Q-learning", "DQN", "Double DQN", "Rainbow", "C51"],
"Policy-Based": ["REINFORCE", "TRPO", "PPO", "A3C", "DDPG"],
"Actor-Critic": ["A2C", "A3C", "SAC", "TD3", "SAC"],
"Model-Based": ["Dyna", "MBMF", "MuZero", "Dreamer", "PlaNet"],
"Hierarchical": ["HIRO", "HAC", "Option-Critic", "MAXQ"],
"Multi-Agent": ["MADDPG", "QMIX", "COMA", "VDN"],
"Inverse RL": ["MaxEnt IRL", "GAIL", "AIRL", "FAIRL"],
"Meta-RL": ["MAML", "RL²", "PEARL", "VariBAD"]
}
# Create radar chart
categories = list(algorithms.keys())
N = len(categories)
# Assign scores for maturity/complexity
scores = {
"Value-Based": 9,
"Policy-Based": 8,
"Actor-Critic": 9,
"Model-Based": 7,
"Hierarchical": 6,
"Multi-Agent": 7,
"Inverse RL": 5,
"Meta-RL": 4
}
values = [scores[cat] for cat in categories]
values += values[:1] # Close the radar chart
angles = np.linspace(0, 2 * np.pi, N, endpoint=False).tolist()
angles += angles[:1]
fig, ax = plt.subplots(figsize=(10, 10), subplot_kw=dict(projection='polar'))
# Draw the plot
ax.plot(angles, values, 'o-', linewidth=2, label='Maturity/Complexity')
ax.fill(angles, values, alpha=0.25)
# Add labels
ax.set_xticks(angles[:-1])
ax.set_xticklabels(categories, fontsize=10)
ax.set_ylim(0, 10)
ax.set_yticks([2, 4, 6, 8, 10])
ax.set_yticklabels(['2', '4', '6', '8', '10'], fontsize=8)
ax.grid(True)
# Add title
ax.set_title('RL Algorithm Landscape (Maturity & Complexity)',
size=15, weight='bold', ha='center', pad=20)
plt.tight_layout()
plt.show()
# Print key insights
print("\n" + "="*80)
print("KEY INSIGHTS FROM RL LANDSCAPE:")
print("="*80)
insights = [
("Value-Based", "Most mature, good for discrete actions"),
("Actor-Critic", "Best balance for continuous control"),
("Model-Based", "Most sample-efficient, but complex"),
("Multi-Agent", "Growing importance for real-world systems"),
("Meta-RL", "Cutting edge, learns to learn quickly")
]
for category, insight in insights:
print(f"• {category:15s}: {insight}")
def explain_modern_algorithms(self):
"""Explain key modern algorithms"""
print("\n" + "="*80)
print("MODERN RL ALGORITHMS EXPLAINED")
print("="*80)
algorithms = [
{
"name": "PPO (Proximal Policy Optimization)",
"key_idea": "Clipped objective prevents too-large policy updates",
"formula": "L^CLIP(θ) = E[min(r_t(θ)A_t, clip(r_t(θ), 1-ε, 1+ε)A_t)]",
"strengths": ["Simple", "Stable", "Good performance"],
"use_cases": ["Robotics", "Games", "Continuous control"]
},
{
"name": "SAC (Soft Actor-Critic)",
"key_idea": "Maximum entropy RL - maximize reward + entropy",
"formula": "J(π) = Σ E[r(s,a) + αH(π(·|s))]",
"strengths": ["Sample efficient", "Stable", "Automated temp tuning"],
"use_cases": ["Robotics", "Autonomous vehicles"]
},
{
"name": "TD3 (Twin Delayed DDPG)",
"key_idea": "Addresses overestimation in actor-critic",
"formula": "Two critics + delayed policy updates + target smoothing",
"strengths": ["Reduces overestimation", "More stable than DDPG"],
"use_cases": ["Continuous control with deterministic policies"]
},
{
"name": "MuZero",
"key_idea": "Learns model, uses it for planning",
"formula": "Learns: dynamics, reward, value functions",
"strengths": ["Model-based efficiency", "No environment knowledge"],
"use_cases": ["Games", "Planning problems"]
},
{
"name": "Decision Transformer",
"key_idea": "RL as sequence modeling problem",
"formula": "P(a_t | s_t, a_{<t}, r_{<t}) - Transformer architecture",
"strengths": ["Leverages transformer power", "Offline RL friendly"],
"use_cases": ["Language tasks", "Recommendation systems"]
}
]
for i, algo in enumerate(algorithms, 1):
print(f"\n{i}. {algo['name']}")
print(f" Key Idea: {algo['key_idea']}")
print(f" Formula: {algo['formula']}")
print(f" Strengths: {', '.join(algo['strengths'])}")
print(f" Use Cases: {', '.join(algo['use_cases'])}")
print("-"*60)
# Initialize frontier exploration
frontier = RLFrontier()
frontier.visualize_algorithm_landscape()
frontier.explain_modern_algorithms()class RealWorldChallenges:
"""Understanding real-world RL challenges and solutions"""
def __init__(self):
self.challenges = self.identify_challenges()
def identify_challenges(self):
"""Identify key real-world RL challenges"""
print("\n" + "="*80)
print("10.2 REAL-WORLD RL CHALLENGES")
print("="*80)
print("\nRL research often focuses on ideal conditions, but real-world")
print("applications face unique challenges:")
challenges = [
{
"name": "Sample Efficiency",
"description": "Real-world interactions are expensive/slow",
"example": "Robot needs thousands of trials to learn simple task",
"solutions": ["Model-based RL", "Sim-to-real", "Offline RL", "Meta-learning"]
},
{
"name": "Safety & Constraints",
"description": "Cannot explore unsafe actions in real world",
"example": "Self-driving car can't crash during learning",
"solutions": ["Safe exploration", "Constraint optimization", "Human oversight"]
},
{
"name": "Reward Specification",
"description": "Hard to design reward functions that capture intent",
"example": "Reward hacking - agent finds loopholes",
"solutions": ["Inverse RL", "Preference learning", "Multi-objective RL"]
},
{
"name": "Non-stationarity",
"description": "Real world changes while agent learns",
"example": "Environment dynamics change over time",
"solutions": ["Continual learning", "Adaptation", "Meta-RL"]
},
{
"name": "Partial Observability",
"description": "Agent only sees part of true state",
"example": "Robot sees limited camera view",
"solutions": ["Memory networks", "Recurrent policies", "State estimation"]
},
{
"name": "Multi-agent Coordination",
"description": "Multiple agents learning simultaneously",
"example": "Autonomous vehicles coordinating",
"solutions": ["Centralized training", "Communication protocols", "Game theory"]
},
{
"name": "Sim-to-Real Gap",
"description": "Simulators don't match reality perfectly",
"example": "Physics simulators have approximations",
"solutions": ["Domain randomization", "System identification", "Adaptation"]
},
{
"name": "Scalability",
"description": "Algorithms don't scale to complex real problems",
"example": "High-dimensional state/action spaces",
"solutions": ["Hierarchical RL", "Skill learning", "Modular architectures"]
}
]
for i, challenge in enumerate(challenges, 1):
print(f"\n{i}. {challenge['name'].upper()}")
print(f" Description: {challenge['description']}")
print(f" Example: {challenge['example']}")
print(f" Solutions: {', '.join(challenge['solutions'])}")
return challenges
def demonstrate_safety_challenge(self):
"""Demonstrate safety challenge in RL"""
print("\n" + "="*80)
print("SAFETY CHALLENGE DEMONSTRATION")
print("="*80)
class UnsafeGridWorld:
"""Grid world with unsafe states"""
def __init__(self, size=5):
self.size = size
self.goal = (4, 4)
self.unsafe = [(1, 1), (2, 2), (3, 1), (4, 0)]
self.state = (0, 0)
self.actions = [(0, 1), (1, 0), (0, -1), (-1, 0)] # right, down, left, up
def reset(self):
self.state = (0, 0)
return self.state
def step(self, action):
r, c = self.state
dr, dc = self.actions[action]
new_r = max(0, min(self.size-1, r + dr))
new_c = max(0, min(self.size-1, c + dc))
self.state = (new_r, new_c)
# Check unsafe state
if self.state in self.unsafe:
reward = -100 # Large penalty for unsafe state
done = True
elif self.state == self.goal:
reward = 10
done = True
else:
reward = -0.1 # Small penalty for each step
done = False
return self.state, reward, done
def render(self):
grid = [['.' for _ in range(self.size)] for _ in range(self.size)]
for r, c in self.unsafe:
grid[r][c] = 'X'
grid[self.goal[0]][self.goal[1]] = 'G'
r, c = self.state
grid[r][c] = 'A'
for row in grid:
print(' '.join(row))
print()
print("\n1. UNSAFE GRID WORLD:")
print(" X = Unsafe state (-100 reward)")
print(" G = Goal (+10 reward)")
print(" . = Normal state (-0.1 per step)")
env = UnsafeGridWorld()
print("\n Initial state:")
env.render()
print("\n2. NAIVE RL APPROACH (without safety):")
print(" Risk: Agent might explore unsafe states during training")
print(" Consequence: Catastrophic failure in real world")
print("\n3. SAFE RL SOLUTIONS:")
solutions = [
("Constrained Policy Optimization",
"Maximize reward subject to safety constraints"),
("Shielded Reinforcement Learning",
"External safety module prevents unsafe actions"),
("Risk-Sensitive RL",
"Optimize conditional value at risk (CVaR)"),
("Human-in-the-loop",
"Human prevents unsafe actions during training")
]
for name, description in solutions:
print(f" • {name}: {description}")
# Demonstrate safe vs unsafe policies
print("\n4. POLICY COMPARISON:")
# Unsafe policy (shortest path through unsafe states)
unsafe_path = [(0,0), (1,0), (2,0), (3,0), (4,0), (4,1), (4,2), (4,3), (4,4)]
# Safe policy (avoids unsafe states)
safe_path = [(0,0), (0,1), (0,2), (0,3), (0,4),
(1,4), (2,4), (3,4), (4,4)]
print(f"\n Unsafe policy (shortest path):")
print(f" Length: {len(unsafe_path)} steps")
print(f" Risk: Goes through unsafe state (4,0)")
print(f"\n Safe policy (avoids unsafe states):")
print(f" Length: {len(safe_path)} steps")
print(f" Risk: No unsafe states")
# Visualize paths
fig, axes = plt.subplots(1, 2, figsize=(12, 5))
for idx, (path, title) in enumerate([(unsafe_path, "Unsafe Policy"),
(safe_path, "Safe Policy")]):
ax = axes[idx]
# Create grid
for r in range(5):
for c in range(5):
if (r, c) in env.unsafe:
color = 'red'
elif (r, c) == env.goal:
color = 'green'
elif (r, c) == (0, 0):
color = 'blue'
else:
color = 'white'
ax.add_patch(plt.Rectangle((c-0.5, r-0.5), 1, 1,
facecolor=color, alpha=0.3,
edgecolor='black'))
# Plot path
if path:
path_array = np.array(path)
ax.plot(path_array[:, 1], path_array[:, 0], 'b-o',
linewidth=2, markersize=8, alpha=0.7)
ax.scatter([path_array[0, 1]], [path_array[0, 0]],
color='blue', s=200, marker='o', label='Start')
ax.scatter([path_array[-1, 1]], [path_array[-1, 0]],
color='green', s=200, marker='*', label='Goal')
ax.set_xlim(-0.5, 4.5)
ax.set_ylim(-0.5, 4.5)
ax.invert_yaxis()
ax.set_aspect('equal')
ax.set_title(title, fontsize=14, fontweight='bold')
ax.set_xlabel('Column')
ax.set_ylabel('Row')
ax.legend()
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\n5. KEY TAKEAWAYS:")
print(" • Safety must be considered from the beginning")
print(" • Unsafe exploration is unacceptable in real systems")
print(" • Constrained optimization is often necessary")
print(" • Human oversight can bridge the gap during learning")
def demonstrate_sim2real(self):
"""Demonstrate sim-to-real transfer challenge"""
print("\n" + "="*80)
print("SIM-TO-REAL TRANSFER CHALLENGE")
print("="*80)
print("\nTHE PROBLEM:")
print("Simulators are imperfect models of reality:")
print("• Approximate physics")
print("• Simplified sensor models")
print("• Missing environmental factors")
print("\n" + "-"*80)
print("DOMAIN RANDOMIZATION SOLUTION:")
print("-"*80)
print("\nIdea: Train in many randomized simulations")
print(" so agent learns robust policy")
# Simulate domain randomization
np.random.seed(42)
# Parameters that vary between sim and real
parameters = {
'friction': {'sim': 0.8, 'real': 1.2},
'mass': {'sim': 1.0, 'real': 1.3},
'sensor_noise': {'sim': 0.01, 'real': 0.05},
'actuator_delay': {'sim': 0.0, 'real': 0.02},
'wind': {'sim': 0.0, 'real': 0.5}
}
print("\nParameter variations:")
for param, values in parameters.items():
print(f" {param:15s}: Sim={values['sim']:.3f}, Real={values['real']:.3f}")
# Demonstrate domain randomization
print("\n" + "-"*80)
print("DOMAIN RANDOMIZATION TRAINING:")
print("-"*80)
num_training_episodes = 5
randomization_ranges = {
'friction': (0.5, 1.5),
'mass': (0.7, 1.5),
'sensor_noise': (0.0, 0.1),
'actuator_delay': (0.0, 0.05),
'wind': (0.0, 1.0)
}
print("\nTraining with randomized parameters:")
for i in range(num_training_episodes):
episode_params = {}
for param, (low, high) in randomization_ranges.items():
episode_params[param] = np.random.uniform(low, high)
param_str = ', '.join([f"{k}:{v:.2f}" for k, v in episode_params.items()])
print(f" Episode {i+1}: {param_str}")
# Visualize the concept
fig, axes = plt.subplots(2, 3, figsize=(15, 10))
# Plot 1: Parameter distributions
ax = axes[0, 0]
params = list(randomization_ranges.keys())
sim_values = [parameters[p]['sim'] for p in params]
real_values = [parameters[p]['real'] for p in params]
x = np.arange(len(params))
width = 0.35
ax.bar(x - width/2, sim_values, width, label='Simulator', alpha=0.7)
ax.bar(x + width/2, real_values, width, label='Reality', alpha=0.7)
ax.set_xlabel('Parameter')
ax.set_ylabel('Value')
ax.set_title('Simulator vs Reality Parameters')
ax.set_xticks(x)
ax.set_xticklabels(params, rotation=45)
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 2: Domain randomization ranges
ax = axes[0, 1]
for i, (param, (low, high)) in enumerate(randomization_ranges.items()):
ax.plot([low, high], [i, i], 'o-', linewidth=3, markersize=8)
ax.scatter([parameters[param]['sim']], [i], color='blue', s=100,
label='Sim' if i == 0 else '', zorder=5)
ax.scatter([parameters[param]['real']], [i], color='red', s=100,
label='Real' if i == 0 else '', zorder=5)
ax.set_yticks(range(len(params)))
ax.set_yticklabels(params)
ax.set_xlabel('Parameter Value')
ax.set_title('Domain Randomization Ranges')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 3: Performance comparison
ax = axes[0, 2]
methods = ['No Rand', 'Limited Rand', 'Full Rand']
sim_performance = [95, 85, 75] # Performance in simulator
real_performance = [30, 65, 90] # Performance in reality
x = np.arange(len(methods))
ax.bar(x - 0.2, sim_performance, 0.4, label='Simulator', alpha=0.7)
ax.bar(x + 0.2, real_performance, 0.4, label='Reality', alpha=0.7)
ax.set_xlabel('Training Method')
ax.set_ylabel('Success Rate (%)')
ax.set_title('Sim-to-Real Transfer Performance')
ax.set_xticks(x)
ax.set_xticklabels(methods)
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 4: Robustness visualization
ax = axes[1, 0]
# Simulate policy robustness
np.random.seed(42)
n_samples = 100
# Non-robust policy
non_robust_rewards = np.random.normal(80, 20, n_samples)
non_robust_rewards = np.clip(non_robust_rewards, 0, 100)
# Robust policy (domain randomization)
robust_rewards = np.random.normal(85, 10, n_samples)
robust_rewards = np.clip(robust_rewards, 0, 100)
ax.boxplot([non_robust_rewards, robust_rewards],
labels=['No Randomization', 'With Randomization'])
ax.set_ylabel('Reward')
ax.set_title('Policy Robustness to Parameter Variations')
ax.grid(True, alpha=0.3)
# Plot 5: Adaptation over time
ax = axes[1, 1]
# Simulate adaptation process
episodes = np.arange(1, 101)
# Without adaptation
no_adapt = 30 + 70 * (1 - np.exp(-episodes / 50))
# With adaptation
with_adapt = 50 + 50 * (1 - np.exp(-episodes / 20))
ax.plot(episodes, no_adapt, 'r-', label='No Adaptation', linewidth=2)
ax.plot(episodes, with_adapt, 'g-', label='With Adaptation', linewidth=2)
ax.set_xlabel('Episode')
ax.set_ylabel('Real-World Performance')
ax.set_title('Adaptation to Reality Over Time')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 6: System diagram
ax = axes[1, 2]
ax.axis('off')
# Draw system diagram
components = [
("Simulator\n(Domain Randomization)", (0.3, 0.7)),
("RL Agent\n(Training)", (0.3, 0.5)),
("Policy\n(Deployment)", (0.3, 0.3)),
("Real World\n(Testing)", (0.7, 0.5))
]
# Draw boxes
for text, (x, y) in components:
ax.add_patch(plt.Rectangle((x-0.15, y-0.1), 0.3, 0.2,
facecolor='lightblue', alpha=0.5,
edgecolor='black'))
ax.text(x, y, text, ha='center', va='center', fontsize=9)
# Draw arrows
arrows = [
((0.3, 0.6), (0.3, 0.4)), # Training to Policy
((0.3, 0.3), (0.7, 0.5)), # Policy to Real World
((0.7, 0.45), (0.3, 0.55)), # Feedback loop
]
for (x1, y1), (x2, y2) in arrows:
ax.annotate('', xy=(x2, y2), xytext=(x1, y1),
arrowprops=dict(arrowstyle='->', lw=2))
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_title('Sim-to-Real Pipeline', fontsize=12)
plt.tight_layout()
plt.show()
print("\nKEY INSIGHTS:")
print("1. Domain randomization creates robust policies")
print("2. Tradeoff: Lower simulator performance for better real-world transfer")
print("3. Adaptive policies can improve during real-world deployment")
print("4. System identification helps bridge the sim-to-real gap")
# Run real-world challenges demonstration
print("\n" + "="*80)
print("DEMONSTRATING REAL-WORLD CHALLENGES")
print("="*80)
real_world = RealWorldChallenges()
real_world.demonstrate_safety_challenge()
real_world.demonstrate_sim2real()class MultiAgentRL:
"""Multi-Agent Reinforcement Learning"""
def __init__(self):
print("\n" + "="*80)
print("10.3 MULTI-AGENT REINFORCEMENT LEARNING")
print("="*80)
self.introduce_marl()
def introduce_marl(self):
"""Introduce Multi-Agent RL concepts"""
print("\nWHY MULTI-AGENT RL MATTERS:")
print("Real-world systems often involve multiple autonomous agents:")
print("• Autonomous vehicles on roads")
print("• Robots in warehouses")
print("• Trading algorithms in markets")
print("• Network routing algorithms")
print("\n" + "-"*80)
print("KEY CHALLENGES:")
print("-"*80)
challenges = [
("Non-stationarity", "Other agents are learning too"),
("Credit assignment", "Who gets credit for team success?"),
("Communication", "How should agents share information?"),
("Coordination", "Avoiding conflicts, achieving cooperation"),
("Scalability", "State space grows exponentially with agents")
]
for challenge, description in challenges:
print(f"• {challenge:20s}: {description}")
def demonstrate_coordination_games(self):
"""Demonstrate coordination problems in multi-agent systems"""
print("\n" + "="*80)
print("COORDINATION GAMES DEMONSTRATION")
print("="*80)
# Define classic coordination games
games = {
"Prisoner's Dilemma": {
"payoffs": np.array([[[3,3], [0,5]],
[[5,0], [1,1]]]),
"description": "Individual rationality vs collective optimal"
},
"Stag Hunt": {
"payoffs": np.array([[[4,4], [0,3]],
[[3,0], [2,2]]]),
"description": "Risk dominance vs payoff dominance"
},
"Battle of the Sexes": {
"payoffs": np.array([[[2,1], [0,0]],
[[0,0], [1,2]]]),
"description": "Coordination with conflicting preferences"
},
"Pure Coordination": {
"payoffs": np.array([[[1,1], [0,0]],
[[0,0], [1,1]]]),
"description": "Simple coordination problem"
}
}
print("\nCLASSIC COORDINATION GAMES:")
for game_name, game_info in games.items():
print(f"\n{game_name}:")
print(f" {game_info['description']}")
print(f" Payoff matrix (row=Agent1, col=Agent2):")
print(f" {game_info['payoffs'][0,0]} | {game_info['payoffs'][0,1]}")
print(f" {game_info['payoffs'][1,0]} | {game_info['payoffs'][1,1]}")
# Simulate learning in these games
print("\n" + "-"*80)
print("LEARNING DYNAMICS IN COORDINATION GAMES")
print("-"*80)
class MultiAgentLearning:
"""Simple multi-agent Q-learning"""
def __init__(self, payoff_matrix, learning_rate=0.1, discount=0.9, epsilon=0.1):
self.payoff_matrix = payoff_matrix
self.lr = learning_rate
self.gamma = discount
self.epsilon = epsilon
# Initialize Q-tables for each agent
self.q1 = np.zeros((2, 2)) # Agent 1: Q(state, action)
self.q2 = np.zeros((2, 2)) # Agent 2: Q(state, action)
# Keep history
self.actions_history = []
self.rewards_history = []
def select_action(self, agent, state):
"""Epsilon-greedy action selection"""
if np.random.random() < self.epsilon:
return np.random.randint(2)
else:
if agent == 1:
return np.argmax(self.q1[state])
else:
return np.argmax(self.q2[state])
def learn_step(self):
"""One learning step"""
# Agents choose actions
a1 = self.select_action(1, 0)
a2 = self.select_action(2, 0)
# Get rewards
r1 = self.payoff_matrix[a1, a2, 0]
r2 = self.payoff_matrix[a1, a2, 1]
# Q-learning updates
# For simplicity, assume next state is always 0
next_state = 0
# Agent 1 update
best_next_q1 = np.max(self.q1[next_state])
td_error1 = r1 + self.gamma * best_next_q1 - self.q1[0, a1]
self.q1[0, a1] += self.lr * td_error1
# Agent 2 update
best_next_q2 = np.max(self.q2[next_state])
td_error2 = r2 + self.gamma * best_next_q2 - self.q2[0, a2]
self.q2[0, a2] += self.lr * td_error2
# Store history
self.actions_history.append((a1, a2))
self.rewards_history.append((r1, r2))
return (a1, a2), (r1, r2)
# Run learning for different games
fig, axes = plt.subplots(2, 2, figsize=(12, 10))
axes = axes.flatten()
for idx, (game_name, game_info) in enumerate(games.items()):
ax = axes[idx]
# Run learning
learner = MultiAgentLearning(game_info['payoffs'])
n_steps = 1000
action_counts = np.zeros((2, 2)) # Count (a1, a2) pairs
for step in range(n_steps):
(a1, a2), _ = learner.learn_step()
action_counts[a1, a2] += 1
# Plot action distribution
im = ax.imshow(action_counts / n_steps, cmap='YlOrRd', vmin=0, vmax=1)
# Add text annotations
for i in range(2):
for j in range(2):
prob = action_counts[i, j] / n_steps
color = 'white' if prob > 0.5 else 'black'
ax.text(j, i, f'{prob:.2f}', ha='center', va='center',
color=color, fontsize=12, fontweight='bold')
ax.set_xticks([0, 1])
ax.set_yticks([0, 1])
ax.set_xticklabels(['C', 'D'] if game_name == "Prisoner's Dilemma" else ['A', 'B'])
ax.set_yticklabels(['C', 'D'] if game_name == "Prisoner's Dilemma" else ['A', 'B'])
ax.set_xlabel('Agent 2 Action')
ax.set_ylabel('Agent 1 Action')
ax.set_title(f'{game_name}\nFinal Action Distribution')
# Add payoff information
payoff_text = f'Payoffs:\n'
for i in range(2):
for j in range(2):
payoff_text += f'({game_info["payoffs"][i,j,0]},{game_info["payoffs"][i,j,1]}) '
payoff_text += '\n'
ax.text(1.5, 0.5, payoff_text, transform=ax.transAxes,
fontsize=8, verticalalignment='center')
plt.colorbar(im, ax=axes[-1])
plt.tight_layout()
plt.show()
print("\nOBSERVED LEARNING DYNAMICS:")
print("1. Prisoner's Dilemma: Often converges to (Defect, Defect) - suboptimal")
print("2. Stag Hunt: Can get stuck in risk-dominant equilibrium")
print("3. Battle of Sexes: May oscillate or need communication")
print("4. Pure Coordination: Usually finds coordination but which one?")
print("\n" + "-"*80)
print("SOLUTIONS FOR MULTI-AGENT COORDINATION:")
print("-"*80)
solutions = [
("Centralized Training", "Train agents together with shared information"),
("Communication Protocols", "Agents learn to send/receive messages"),
("Credit Assignment", "Properly assign credit in team settings"),
("Opponent Modeling", "Agents learn models of other agents"),
("Mechanism Design", "Design rewards to encourage cooperation")
]
for solution, description in solutions:
print(f"• {solution:25s}: {description}")
def demonstrate_maddpg(self):
"""Demonstrate MADDPG algorithm"""
print("\n" + "="*80)
print("MADDPG: MULTI-AGENT DDPG")
print("="*80)
print("\nMADDPG (Multi-Agent Deep Deterministic Policy Gradient)")
print("Key Idea: Centralized training with decentralized execution")
print(" Each agent has its own actor, but critics see all agents' info")
class MADDPGComponents:
"""Visualize MADDPG architecture"""
def __init__(self, n_agents=3, state_dim=4, action_dim=2):
self.n_agents = n_agents
self.state_dim = state_dim
self.action_dim = action_dim
def visualize_architecture(self):
"""Visualize MADDPG architecture"""
fig, axes = plt.subplots(1, 2, figsize=(15, 6))
# Left: Centralized Critic during training
ax1 = axes[0]
ax1.set_title('CENTRALIZED TRAINING PHASE', fontsize=14, fontweight='bold')
ax1.axis('off')
# Draw agents
agent_y_positions = [0.8, 0.5, 0.2]
agent_colors = ['#FF6B6B', '#4ECDC4', '#FFD166']
for i, (y, color) in enumerate(zip(agent_y_positions, agent_colors)):
# Actor network
ax1.add_patch(plt.Rectangle((0.1, y-0.05), 0.2, 0.1,
facecolor=color, alpha=0.7,
edgecolor='black'))
ax1.text(0.2, y, f'Actor {i+1}\nπ_i',
ha='center', va='center', fontsize=10)
# Input arrows
ax1.annotate('', xy=(0.1, y), xytext=(0.05, y),
arrowprops=dict(arrowstyle='->', lw=1))
ax1.text(0.025, y, f's_i', ha='right', va='center')
# Output arrows
ax1.annotate('', xy=(0.3, y), xytext=(0.35, y),
arrowprops=dict(arrowstyle='->', lw=1))
ax1.text(0.375, y, f'a_i', ha='left', va='center')
# Centralized critic
ax1.add_patch(plt.Rectangle((0.4, 0.3), 0.2, 0.4,
facecolor='#06D6A0', alpha=0.7,
edgecolor='black'))
ax1.text(0.5, 0.5, 'Centralized Critic\nQ(s, a₁, a₂, a₃)',
ha='center', va='center', fontsize=10, fontweight='bold')
# Inputs to critic
ax1.text(0.3, 0.8, 's₁', ha='right', va='center')
ax1.text(0.3, 0.5, 's₂', ha='right', va='center')
ax1.text(0.3, 0.2, 's₃', ha='right', va='center')
ax1.text(0.35, 0.8, 'a₁', ha='right', va='center')
ax1.text(0.35, 0.5, 'a₂', ha='right', va='center')
ax1.text(0.35, 0.2, 'a₃', ha='right', va='center')
# Arrows to critic
for y in agent_y_positions:
ax1.annotate('', xy=(0.4, y), xytext=(0.35, y),
arrowprops=dict(arrowstyle='->', lw=1))
# Output from critic
ax1.annotate('', xy=(0.65, 0.5), xytext=(0.6, 0.5),
arrowprops=dict(arrowstyle='->', lw=2))
ax1.text(0.7, 0.5, 'Q-value\n(criticizes all)',
ha='left', va='center')
# Right: Decentralized execution
ax2 = axes[1]
ax2.set_title('DECENTRALIZED EXECUTION PHASE', fontsize=14, fontweight='bold')
ax2.axis('off')
for i, (y, color) in enumerate(zip(agent_y_positions, agent_colors)):
# Actor network (deployed)
ax2.add_patch(plt.Rectangle((0.3, y-0.05), 0.2, 0.1,
facecolor=color, alpha=0.7,
edgecolor='black'))
ax2.text(0.4, y, f'Actor {i+1}\nπ_i',
ha='center', va='center', fontsize=10)
# Only local observation
ax2.annotate('', xy=(0.3, y), xytext=(0.1, y),
arrowprops=dict(arrowstyle='->', lw=1))
ax2.text(0.05, y, f'Local obs\ns_i', ha='right', va='center')
# Action output
ax2.annotate('', xy=(0.5, y), xytext=(0.7, y),
arrowprops=dict(arrowstyle='->', lw=1))
ax2.text(0.75, y, f'Action\na_i', ha='left', va='center')
plt.tight_layout()
plt.show()
print("\nMADDPG ALGORITHM STEPS:")
steps = [
("1. Centralized Critic", "Each agent's critic sees all states and actions"),
("2. Policy Gradient", "Actors updated using centralized critic"),
("3. Target Networks", "Slow-updating target networks for stability"),
("4. Experience Replay", "Store transitions from all agents"),
("5. Decentralized Execution", "Each agent uses only its own actor")
]
for step, description in steps:
print(f"{step:25s}: {description}")
# Show MADDPG architecture
maddpg_viz = MADDPGComponents()
maddpg_viz.visualize_architecture()
# Demonstrate MADDPG benefits
print("\n" + "="*80)
print("MADDPG BENEFITS AND RESULTS")
print("="*80)
# Simulated results comparison
fig, axes = plt.subplots(2, 2, figsize=(12, 10))
# Plot 1: Learning curves comparison
ax = axes[0, 0]
episodes = np.arange(1000)
# Simulated learning curves
independent_ddpg = 0.3 * (1 - np.exp(-episodes / 200)) + 0.1 * np.random.randn(1000)
independent_ddpg = np.cumsum(np.clip(independent_ddpg, 0, None))
maddpg_performance = 0.5 * (1 - np.exp(-episodes / 100)) + 0.05 * np.random.randn(1000)
maddpg_performance = np.cumsum(np.clip(maddpg_performance, 0, None))
ax.plot(episodes, independent_ddpg, 'r-', label='Independent DDPG', alpha=0.7)
ax.plot(episodes, maddpg_performance, 'g-', label='MADDPG', alpha=0.7)
ax.fill_between(episodes, independent_ddpg*0.9, independent_ddpg*1.1, alpha=0.2, color='red')
ax.fill_between(episodes, maddpg_performance*0.95, maddpg_performance*1.05, alpha=0.2, color='green')
ax.set_xlabel('Episode')
ax.set_ylabel('Cumulative Reward')
ax.set_title('Learning Curve Comparison')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 2: Cooperation metrics
ax = axes[0, 1]
cooperation_scenarios = ['Predator-Prey', 'Multi-Walker', 'Cooperative Navigation']
independent_scores = [0.4, 0.3, 0.35]
maddpg_scores = [0.85, 0.75, 0.9]
x = np.arange(len(cooperation_scenarios))
width = 0.35
ax.bar(x - width/2, independent_scores, width, label='Independent', alpha=0.7)
ax.bar(x + width/2, maddpg_scores, width, label='MADDPG', alpha=0.7)
ax.set_xlabel('Environment')
ax.set_ylabel('Cooperation Score')
ax.set_title('Coordination Performance')
ax.set_xticks(x)
ax.set_xticklabels(cooperation_scenarios, rotation=15)
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 3: Scalability analysis
ax = axes[1, 0]
n_agents = [2, 4, 8, 16]
independent_times = [100, 250, 800, 3000]
maddpg_times = [120, 180, 300, 600]
ax.plot(n_agents, independent_times, 'ro-', label='Independent DDPG', markersize=8)
ax.plot(n_agents, maddpg_times, 'go-', label='MADDPG', markersize=8)
ax.set_xlabel('Number of Agents')
ax.set_ylabel('Training Time (relative)')
ax.set_title('Scalability Analysis')
ax.set_xscale('log')
ax.set_yscale('log')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 4: Communication patterns
ax = axes[1, 1]
ax.axis('off')
# Draw communication patterns
communication_types = [
("No Communication", 0.1, 'red'),
("Broadcast", 0.4, 'orange'),
("Learned Protocols", 0.7, 'yellow'),
("Attention-Based", 0.9, 'green')
]
for i, (name, score, color) in enumerate(communication_types):
y = 0.8 - i * 0.2
ax.add_patch(plt.Rectangle((0.2, y-0.05), score, 0.1,
facecolor=color, alpha=0.7,
edgecolor='black'))
ax.text(0.15, y, name, ha='right', va='center', fontsize=10)
ax.text(0.2 + score + 0.05, y, f'{score:.1f}',
ha='left', va='center', fontsize=10)
ax.text(0.5, 0.9, 'Communication Effectiveness',
ha='center', va='center', fontsize=12, fontweight='bold')
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
plt.tight_layout()
plt.show()
print("\nKEY ADVANCES IN MULTI-AGENT RL:")
print("-"*80)
advances = [
("QMIX", "Value decomposition for cooperative tasks"),
("COMA", "Counterfactual multi-agent policy gradients"),
("CommNet", "Learned communication between agents"),
("TarMAC", "Targeted multi-agent communication"),
("LOLA", "Learning with opponent-learning awareness"),
("PBT", "Population-based training for diverse strategies")
]
for advance, description in advances:
print(f"• {advance:15s}: {description}")
# Run multi-agent RL demonstration
print("\n" + "="*80)
print("DEMONSTRATING MULTI-AGENT RL")
print("="*80)
multi_agent = MultiAgentRL()
multi_agent.demonstrate_coordination_games()
multi_agent.demonstrate_maddpg()class MetaReinforcementLearning:
"""Meta-Learning: Learning to Learn"""
def __init__(self):
print("\n" + "="*80)
print("10.4 META-REINFORCEMENT LEARNING")
print("="*80)
self.introduce_meta_rl()
def introduce_meta_rl(self):
"""Introduce Meta-Reinforcement Learning"""
print("\nTHE CORE IDEA:")
print("Instead of learning a specific task...")
print("Learn HOW TO LEARN new tasks quickly!")
print("\n" + "-"*80)
print("ANALOGY:")
print("-"*80)
print("Standard RL: Learning to play a specific video game")
print("Meta-RL: Learning how to quickly learn ANY new video game")
print("\n" + "-"*80)
print("KEY CONCEPTS:")
print("-"*80)
concepts = [
("Task Distribution", "Set of related tasks to meta-learn from"),
("Inner Loop", "Fast adaptation to a new task (few-shot learning)"),
("Outer Loop", "Slow learning of adaptation algorithm"),
("Meta-Objective", "Performance after adaptation to new tasks"),
("Context", "Information about current task")
]
for concept, description in concepts:
print(f"• {concept:20s}: {description}")
def demonstrate_maml(self):
"""Demonstrate Model-Agnostic Meta-Learning (MAML)"""
print("\n" + "="*80)
print("MAML: MODEL-AGNOSTIC META-LEARNING")
print("="*80)
print("\nMAML ALGORITHM:")
print("1. Sample batch of tasks from task distribution")
print("2. For each task:")
print(" a. Compute gradient on task's data")
print(" b. Take one gradient step (inner loop)")
print("3. Update parameters to improve performance AFTER adaptation")
print("4. Repeat")
class MAMLVisualizer:
"""Visualize MAML optimization"""
def __init__(self):
self.task_functions = self.create_task_functions()
def create_task_functions(self):
"""Create different tasks (sine waves with different phases)"""
tasks = []
phases = [0, np.pi/2, np.pi, 3*np.pi/2]
for phase in phases:
def task(x, phase=phase):
return np.sin(x + phase)
tasks.append(task)
return tasks
def visualize_maml_optimization(self):
"""Visualize MAML optimization process"""
fig, axes = plt.subplots(2, 2, figsize=(12, 10))
# Generate data
np.random.seed(42)
x = np.linspace(-5, 5, 100)
# Plot 1: Task distribution
ax = axes[0, 0]
for i, task_func in enumerate(self.task_functions):
y = task_func(x)
ax.plot(x, y, label=f'Task {i+1}', alpha=0.7)
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_title('Task Distribution (Different Sine Waves)')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 2: Initial model
ax = axes[0, 1]
# Initial model (poor fit for all tasks)
initial_model = lambda x: 0.5 * np.sin(x)
y_initial = initial_model(x)
ax.plot(x, y_initial, 'k--', linewidth=2, label='Initial Model')
# Show adaptation for one task
task_idx = 0
task_data_x = np.array([-4, -2, 0, 2, 4])
task_data_y = self.task_functions[task_idx](task_data_x)
ax.scatter(task_data_x, task_data_y, color='red', s=50,
label='Task Data', zorder=5)
# One gradient step adaptation
# Simplified: move toward task data
adapted_model = lambda x: 0.5 * np.sin(x) + 0.3 * np.sin(x + np.pi/2)
y_adapted = adapted_model(x)
ax.plot(x, y_adapted, 'g-', linewidth=2, label='After 1-step Adaptation')
ax.set_title('Fast Adaptation (Inner Loop)')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 3: Meta-update visualization
ax = axes[1, 0]
# Create loss landscape visualization
theta1 = np.linspace(-2, 2, 100)
theta2 = np.linspace(-2, 2, 100)
Theta1, Theta2 = np.meshgrid(theta1, theta2)
# Simplified loss function
Loss = np.zeros_like(Theta1)
for i in range(Theta1.shape[0]):
for j in range(Theta2.shape[1]):
# Loss after adaptation to different tasks
losses = []
for task_func in self.task_functions:
# Simulated adaptation
adapted_theta1 = Theta1[i, j] + 0.1 * np.random.randn()
adapted_theta2 = Theta2[i, j] + 0.1 * np.random.randn()
# Simplified loss
loss = (adapted_theta1 - 0.5)**2 + (adapted_theta2 - 0.3)**2
losses.append(loss)
Loss[i, j] = np.mean(losses)
# Plot loss landscape
contour = ax.contourf(Theta1, Theta2, Loss, levels=20, cmap='viridis', alpha=0.8)
plt.colorbar(contour, ax=ax)
# Plot optimization path
path_theta1 = [1.5, 1.0, 0.7, 0.5]
path_theta2 = [1.5, 1.0, 0.8, 0.3]
ax.plot(path_theta1, path_theta2, 'ro-', linewidth=2, markersize=8)
ax.scatter(path_theta1[0], path_theta2[0], color='yellow', s=200,
marker='*', label='Start', zorder=5)
ax.scatter(path_theta1[-1], path_theta2[-1], color='red', s=200,
marker='X', label='MAML Optimum', zorder=5)
ax.set_xlabel('Parameter θ₁')
ax.set_ylabel('Parameter θ₂')
ax.set_title('Meta-Learning Loss Landscape')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 4: Few-shot learning performance
ax = axes[1, 1]
k_shots = [1, 5, 10, 20, 50]
standard_rl = [0.2, 0.4, 0.6, 0.75, 0.9]
meta_rl = [0.6, 0.85, 0.95, 0.98, 0.99]
pretrained = [0.8, 0.9, 0.95, 0.97, 0.98]
ax.plot(k_shots, standard_rl, 'ro-', label='Standard RL', markersize=8)
ax.plot(k_shots, meta_rl, 'go-', label='Meta-RL (MAML)', markersize=8)
ax.plot(k_shots, pretrained, 'bo-', label='Pretrained + Fine-tune', markersize=8)
ax.set_xlabel('Number of Training Examples (k-shot)')
ax.set_ylabel('Test Performance')
ax.set_title('Few-Shot Learning Comparison')
ax.set_xscale('log')
ax.legend()
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\nMAML MATHEMATICAL FORMULATION:")
print("-"*80)
print("Inner loop (task adaptation):")
print(" θ_i' = θ - α ∇_θ L_i(f_θ)")
print("\nOuter loop (meta-update):")
print(" θ ← θ - β ∇_θ Σ_i L_i(f_{θ_i'})")
print("\nWhere:")
print(" α: Inner learning rate (fast adaptation)")
print(" β: Outer learning rate (meta-learning rate)")
print(" L_i: Loss for task i")
print(" θ_i': Task-specific adapted parameters")
# Show MAML visualization
maml_viz = MAMLVisualizer()
maml_viz.visualize_maml_optimization()
print("\n" + "="*80)
print("META-RL APPLICATIONS")
print("="*80)
applications = [
("Robotics", "Quickly adapt to new objects/environments"),
("Personalization", "Adapt to individual user preferences"),
("Continual Learning", "Learn sequences of related tasks"),
("Sim-to-Real", "Adapt quickly to real-world variations"),
("Game Playing", "Quickly learn new games or levels")
]
print("\nREAL-WORLD APPLICATIONS:")
for app, description in applications:
print(f"• {app:20s}: {description}")
print("\n" + "-"*80)
print("META-RL ALGORITHMS:")
print("-"*80)
algorithms = [
("RL²", "Recurrent model that learns learning algorithm"),
("PEARL", "Probabilistic embeddings for actor-critic meta-learning"),
("VariBAD", "Variational Bayes-Adaptive Deep RL"),
("CAVIA", "Fast adaptation via context parameters"),
("ProMP", "Proximal meta-policy search")
]
for algo, description in algorithms:
print(f"• {algo:15s}: {description}")
# Run meta-RL demonstration
print("\n" + "="*80)
print("DEMONSTRATING META-REINFORCEMENT LEARNING")
print("="*80)
meta_rl = MetaReinforcementLearning()
meta_rl.demonstrate_maml()class RLHFExplainer:
"""Reinforcement Learning from Human Feedback"""
def __init__(self):
print("\n" + "="*80)
print("10.5 REINFORCEMENT LEARNING FROM HUMAN FEEDBACK (RLHF)")
print("="*80)
self.introduce_rlhf()
def introduce_rlhf(self):
"""Introduce RLHF"""
print("\nTHE RLHF REVOLUTION:")
print("How ChatGPT, Claude, and other LLMs were trained!")
print("\n" + "-"*80)
print("WHY RLHF?")
print("-"*80)
print("1. Reward functions are hard to specify for complex tasks")
print("2. Humans have nuanced preferences hard to capture in rewards")
print("3. Alignment: Make AI systems helpful, harmless, honest")
print("4. Natural interface: Humans provide feedback naturally")
def demonstrate_rlhf_pipeline(self):
"""Demonstrate the RLHF pipeline"""
print("\n" + "="*80)
print("RLHF PIPELINE FOR LANGUAGE MODELS")
print("="*80)
# Create visualization
fig, axes = plt.subplots(2, 3, figsize=(15, 10))
# Step 1: Supervised Fine-Tuning
ax = axes[0, 0]
ax.axis('off')
ax.text(0.5, 0.8, 'STEP 1:', ha='center', va='center',
fontsize=12, fontweight='bold')
ax.text(0.5, 0.7, 'Supervised Fine-Tuning', ha='center', va='center',
fontsize=11)
# Draw SFT process
ax.add_patch(plt.Rectangle((0.2, 0.4), 0.3, 0.2,
facecolor='lightblue', alpha=0.7,
edgecolor='black'))
ax.text(0.35, 0.5, 'Pre-trained\nLLM', ha='center', va='center')
ax.annotate('', xy=(0.5, 0.5), xytext=(0.65, 0.5),
arrowprops=dict(arrowstyle='->', lw=1))
ax.add_patch(plt.Rectangle((0.65, 0.4), 0.3, 0.2,
facecolor='lightgreen', alpha=0.7,
edgecolor='black'))
ax.text(0.8, 0.5, 'SFT Model', ha='center', va='center')
ax.text(0.35, 0.3, 'Human-written\ndemonstrations',
ha='center', va='center', fontsize=9)
# Step 2: Reward Model Training
ax = axes[0, 1]
ax.axis('off')
ax.text(0.5, 0.8, 'STEP 2:', ha='center', va='center',
fontsize=12, fontweight='bold')
ax.text(0.5, 0.7, 'Reward Model Training', ha='center', va='center',
fontsize=11)
# Draw comparison data collection
ax.text(0.3, 0.6, 'Prompt:', ha='right', va='center')
ax.text(0.35, 0.6, '"Explain RLHF"', ha='left', va='center',
fontsize=9, style='italic')
ax.text(0.3, 0.5, 'Response A:', ha='right', va='center')
ax.text(0.35, 0.5, 'Good explanation...', ha='left', va='center',
fontsize=8, style='italic', color='green')
ax.text(0.3, 0.4, 'Response B:', ha='right', va='center')
ax.text(0.35, 0.4, 'Bad explanation...', ha='left', va='center',
fontsize=8, style='italic', color='red')
ax.text(0.6, 0.45, 'Human\nPreference\n(A > B)',
ha='center', va='center', fontsize=9,
bbox=dict(boxstyle='round', facecolor='yellow', alpha=0.5))
ax.annotate('', xy=(0.7, 0.45), xytext=(0.8, 0.45),
arrowprops=dict(arrowstyle='->', lw=1))
ax.add_patch(plt.Rectangle((0.8, 0.35), 0.15, 0.2,
facecolor='orange', alpha=0.7,
edgecolor='black'))
ax.text(0.875, 0.45, 'Reward\nModel', ha='center', va='center',
fontsize=9)
# Step 3: RL Fine-Tuning
ax = axes[0, 2]
ax.axis('off')
ax.text(0.5, 0.8, 'STEP 3:', ha='center', va='center',
fontsize=12, fontweight='bold')
ax.text(0.5, 0.7, 'RL Fine-Tuning (PPO)', ha='center', va='center',
fontsize=11)
# Draw RL loop
ax.add_patch(plt.Rectangle((0.2, 0.4), 0.2, 0.2,
facecolor='lightgreen', alpha=0.7,
edgecolor='black'))
ax.text(0.3, 0.5, 'Policy\n(LLM)', ha='center', va='center',
fontsize=9)
ax.text(0.1, 0.5, 'Prompt', ha='right', va='center')
ax.annotate('', xy=(0.2, 0.5), xytext=(0.1, 0.5),
arrowprops=dict(arrowstyle='->', lw=1))
ax.text(0.3, 0.3, 'Response', ha='center', va='center')
ax.annotate('', xy=(0.3, 0.4), xytext=(0.3, 0.35),
arrowprops=dict(arrowstyle='->', lw=1))
ax.add_patch(plt.Rectangle((0.5, 0.4), 0.2, 0.2,
facecolor='orange', alpha=0.7,
edgecolor='black'))
ax.text(0.6, 0.5, 'Reward\nModel', ha='center', va='center',
fontsize=9)
ax.text(0.6, 0.3, 'Reward', ha='center', va='center')
ax.annotate('', xy=(0.6, 0.4), xytext=(0.6, 0.35),
arrowprops=dict(arrowstyle='->', lw=1))
ax.annotate('', xy=(0.7, 0.5), xytext=(0.5, 0.5),
arrowprops=dict(arrowstyle='->', lw=1, color='red'))
ax.text(0.6, 0.6, 'Update Policy\nvia PPO', ha='center', va='center',
fontsize=8, color='red')
# Plot 4: Reward model architecture
ax = axes[1, 0]
# Simulate reward model scores
np.random.seed(42)
n_samples = 50
good_responses = np.random.normal(0.8, 0.1, n_samples)
bad_responses = np.random.normal(0.3, 0.2, n_samples)
positions = np.arange(n_samples)
ax.scatter(positions, good_responses, alpha=0.6,
label='Preferred responses', color='green')
ax.scatter(positions, bad_responses, alpha=0.6,
label='Dispreferred responses', color='red')
ax.axhline(y=0.5, color='black', linestyle='--', alpha=0.5)
ax.text(n_samples/2, 0.52, 'Decision Boundary',
ha='center', va='bottom', fontsize=9)
ax.set_xlabel('Sample Index')
ax.set_ylabel('Reward Model Score')
ax.set_title('Reward Model Discrimination')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 5: KL divergence constraint
ax = axes[1, 1]
# Simulate KL divergence during training
episodes = np.arange(100)
kl_divergence = 0.1 + 0.5 * np.exp(-episodes / 20) + 0.05 * np.random.randn(100)
ax.plot(episodes, kl_divergence, 'b-', linewidth=2)
ax.axhline(y=0.15, color='red', linestyle='--', alpha=0.7,
label='KL constraint')
ax.fill_between(episodes, 0, 0.15, alpha=0.1, color='red')
ax.set_xlabel('Training Step')
ax.set_ylabel('KL Divergence')
ax.set_title('KL Constraint in RLHF')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 6: Human preference alignment
ax = axes[1, 2]
aspects = ['Helpfulness', 'Harmlessness', 'Honesty', 'Coherence']
before_rlhf = [0.6, 0.5, 0.7, 0.8]
after_rlhf = [0.9, 0.85, 0.88, 0.85]
x = np.arange(len(aspects))
width = 0.35
ax.bar(x - width/2, before_rlhf, width, label='Before RLHF', alpha=0.7)
ax.bar(x + width/2, after_rlhf, width, label='After RLHF', alpha=0.7)
ax.set_xlabel('Alignment Aspect')
ax.set_ylabel('Human Preference Score')
ax.set_title('RLHF Alignment Improvement')
ax.set_xticks(x)
ax.set_xticklabels(aspects)
ax.legend()
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\n" + "="*80)
print("RLHF KEY COMPONENTS")
print("="*80)
components = [
("Reward Modeling", "Learn human preferences from comparisons"),
("KL Penalty", "Prevent policy from drifting too far from original"),
("PPO Algorithm", "Optimize policy with reward model feedback"),
("Iterative Refinement", "Collect more data, retrain, repeat")
]
for component, description in components:
print(f"• {component:20s}: {description}")
print("\n" + "-"*80)
print("RLHF CHALLENGES:")
print("-"*80)
challenges = [
("Scalability", "Need massive human preference data"),
("Bias", "Human preferences contain biases"),
("Reward Hacking", "Policy learns to exploit reward model"),
("Overoptimization", "Goodhart's law: metric ≠ objective")
]
for challenge, description in challenges:
print(f"• {challenge:20s}: {description}")
print("\n" + "-"*80)
print("ADVANCES BEYOND RLHF:")
print("-"*80)
advances = [
("Constitutional AI", "AI critiques and improves itself"),
("Direct Preference Optimization", "Simpler alternative to RLHF"),
("RAFT", "Rejection sampling fine-tuning"),
("STaR", "Self-taught reasoner")
]
for advance, description in advances:
print(f"• {advance:25s}: {description}")
# Run RLHF demonstration
print("\n" + "="*80)
print("DEMONSTRATING REINFORCEMENT LEARNING FROM HUMAN FEEDBACK")
print("="*80)
rlhf = RLHFExplainer()
rlhf.demonstrate_rlhf_pipeline()class RLFuture:
"""The Future of Reinforcement Learning"""
def __init__(self):
print("\n" + "="*80)
print("10.6 THE FUTURE OF REINFORCEMENT LEARNING")
print("="*80)
self.explore_future_directions()
def explore_future_directions(self):
"""Explore future directions in RL"""
print("\nGRAND CHALLENGES:")
print("-"*80)
challenges = [
("Artificial General Intelligence", "Human-level learning and reasoning"),
("Real-World Autonomy", "Robots that learn complex skills"),
("Scientific Discovery", "AI scientists making breakthroughs"),
("Personal AI Assistants", "Agents that understand and help individuals"),
("AI Alignment", "Ensuring AI systems are safe and beneficial")
]
for i, (challenge, description) in enumerate(challenges, 1):
print(f"{i}. {challenge}:")
print(f" {description}")
print("\n" + "="*80)
print("EMERGING RESEARCH DIRECTIONS")
print("="*80)
# Create future roadmap visualization
fig, axes = plt.subplots(2, 2, figsize=(14, 10))
# Plot 1: Research trends timeline
ax = axes[0, 0]
research_areas = [
("Foundation Models\nfor RL", 2024, 2030),
("World Models\n& Planning", 2023, 2028),
("Causal RL", 2025, 2032),
("Neuro-Symbolic RL", 2024, 2031),
("Lifelong RL", 2023, 2035),
("Embodied AI", 2024, 2033)
]
current_year = 2024
years = np.arange(2023, 2036)
for i, (area, start, end) in enumerate(research_areas):
y_pos = len(research_areas) - i - 1
ax.plot([start, end], [y_pos, y_pos], 'o-', linewidth=3, markersize=8)
ax.text(end + 0.2, y_pos, area, va='center', fontsize=9)
ax.axvline(x=current_year, color='red', linestyle='--', alpha=0.7)
ax.text(current_year, len(research_areas) - 0.5, 'Now',
ha='center', color='red', fontweight='bold')
ax.set_xlabel('Year')
ax.set_title('RL Research Roadmap')
ax.set_yticks([])
ax.grid(True, alpha=0.3, axis='x')
# Plot 2: Capability progression
ax = axes[0, 1]
capabilities = [
"Game Playing\n(Chess, Go)",
"Video Games\n(Atari, Dota)",
"Robotics\n(Manipulation)",
"Autonomous\nDriving",
"Scientific\nDiscovery",
"AGI"
]
difficulty = [1, 3, 5, 7, 9, 10]
current_level = [1, 0.8, 0.5, 0.3, 0.1, 0.01]
x = np.arange(len(capabilities))
ax.bar(x, difficulty, alpha=0.3, color='gray', label='Theoretical Difficulty')
ax.bar(x, [d * c for d, c in zip(difficulty, current_level)],
alpha=0.7, color='blue', label='Current Progress')
ax.set_xlabel('Domain')
ax.set_ylabel('Progress / Difficulty')
ax.set_title('RL Capability Progression')
ax.set_xticks(x)
ax.set_xticklabels(capabilities, rotation=45, ha='right')
ax.legend()
ax.grid(True, alpha=0.3)
# Plot 3: Technology readiness levels
ax = axes[1, 0]
technologies = [
("Model-Based RL", 6),
("Multi-Agent RL", 5),
("Meta-RL", 4),
("Offline RL", 7),
("Hierarchical RL", 5),
("Inverse RL", 4),
("RLHF", 8)
]
tech_names = [t[0] for t in technologies]
trl_levels = [t[1] for t in technologies]
x = np.arange(len(technologies))
colors = plt.cm.RdYlGn(trl_levels / 9)
bars = ax.barh(x, trl_levels, color=colors)
ax.set_xlabel('Technology Readiness Level (1-9)')
ax.set_title('RL Technology Readiness')
ax.set_yticks(x)
ax.set_yticklabels(tech_names)
ax.set_xlim(0, 9)
# Add TRL annotations
trl_descriptions = {
1: 'Basic principles',
3: 'Proof of concept',
5: 'Lab validation',
7: 'System prototype',
9: 'Mission proven'
}
for level, desc in trl_descriptions.items():
ax.axvline(x=level, color='gray', linestyle=':', alpha=0.5)
ax.text(level, -0.5, desc, ha='center', fontsize=8, rotation=45)
# Plot 4: Societal impact areas
ax = axes[1, 1]
impact_areas = [
'Healthcare\n& Medicine',
'Education\n& Tutoring',
'Climate\n& Energy',
'Transportation\n& Logistics',
'Manufacturing\n& Robotics',
'Finance\n& Economics',
'Art\n& Creativity'
]
impact_scores = [9, 8, 9, 8, 7, 6, 5]
timeframe = [5, 3, 10, 5, 4, 3, 2]
scatter = ax.scatter(timeframe, impact_scores, s=200, alpha=0.6,
c=timeframe, cmap='viridis')
for i, area in enumerate(impact_areas):
ax.annotate(area, (timeframe[i], impact_scores[i]),
xytext=(5, 5), textcoords='offset points',
fontsize=9, ha='left')
ax.set_xlabel('Time to Impact (years)')
ax.set_ylabel('Potential Impact (1-10)')
ax.set_title('Societal Impact of RL')
ax.grid(True, alpha=0.3)
plt.colorbar(scatter, ax=ax, label='Years to Impact')
plt.tight_layout()
plt.show()
print("\n" + "="*80)
print("KEY TECHNICAL BREAKTHROUGHS NEEDED")
print("="*80)
breakthroughs = [
("Sample Efficiency", "Learn complex tasks with human-level data"),
("Compositionality", "Combine learned skills in novel ways"),
("Causal Understanding", "Reason about cause and effect"),
("Common Sense", "Background knowledge about the world"),
("Theory of Mind", "Understand other agents' mental states"),
("Safe Exploration", "Learn without catastrophic failures")
]
for breakthrough, description in breakthroughs:
print(f"• {breakthrough:25s}: {description}")
print("\n" + "="*80)
print("PRACTICAL ADVICE FOR RL PRACTITIONERS")
print("="*80)
advice = [
("Start Simple", "Begin with well-understood algorithms (PPO, SAC)"),
("Understand the Basics", "Master fundamentals before cutting-edge"),
("Reproduce Papers", "Build intuition by implementing from scratch"),
("Join Communities", "RL is fast-moving - stay connected"),
("Work on Real Problems", "Theory needs practical validation"),
("Consider Ethics", "Think about societal impact from day one")
]
for item, description in advice:
print(f"• {item:25s}: {description}")
# Explore the future of RL
print("\n" + "="*80)
print("EXPLORING THE FUTURE OF REINFORCEMENT LEARNING")
print("="*80)
future = RLFuture()class CompleteRLSystem:
"""Complete RL System Design Pattern"""
def __init__(self):
print("\n" + "="*80)
print("10.7 PUTTING IT ALL TOGETHER: COMPLETE RL SYSTEM")
print("="*80)
self.design_complete_system()
def design_complete_system(self):
"""Design a complete RL system from scratch"""
print("\nA production-ready RL system has many components:")
class ProductionRLSystem:
"""Blueprint for production RL system"""
def __init__(self):
self.components = self.define_system_components()
def define_system_components(self):
"""Define all components of a production RL system"""
components = {
"Data Pipeline": [
"Experience collection",
"Replay buffers",
"Data augmentation",
"Feature engineering",
"Data validation"
],
"Model Architecture": [
"Network design",
"Normalization layers",
"Regularization",
"Model checkpointing",
"Version control"
],
"Training System": [
"Distributed training",
"Hyperparameter tuning",
"Curriculum learning",
"Monitoring & logging",
"Early stopping"
],
"Evaluation Framework": [
"Test environments",
"A/B testing",
"Statistical significance",
"Performance metrics",
"Safety validation"
],
"Deployment System": [
"Model serving",
"Rollout policies",
"Online learning",
"Fail-safe mechanisms",
"Rollback procedures"
],
"Monitoring & Ops": [
"Performance monitoring",
"Drift detection",
"Alerting systems",
"Resource management",
"Cost optimization"
]
}
return components
def visualize_system_architecture(self):
"""Visualize complete system architecture"""
fig, ax = plt.subplots(figsize=(14, 8))
ax.axis('off')
# Draw system architecture
system_layers = [
("Environment\n& Simulators", 0.1, 0.9, '#FFB74D'),
("Data Collection\nPipeline", 0.3, 0.9, '#4FC3F7'),
("Training\nCluster", 0.5, 0.9, '#81C784'),
("Model Registry\n& Versioning", 0.7, 0.9, '#FF8A65'),
("Serving\nInfrastructure", 0.9, 0.9, '#9575CD')
]
# Draw monitoring and ops layer
ax.add_patch(plt.Rectangle((0.05, 0.05), 0.9, 0.1,
facecolor='#90A4AE', alpha=0.3,
edgecolor='black'))
ax.text(0.5, 0.1, 'MONITORING & OPERATIONS\n'
'Logging | Alerting | Metrics | Dashboards',
ha='center', va='center', fontsize=10)
# Draw main components
for name, x, y, color in system_layers:
# Component box
ax.add_patch(plt.Rectangle((x-0.08, y-0.08), 0.16, 0.16,
facecolor=color, alpha=0.8,
edgecolor='black', linewidth=2))
ax.text(x, y, name, ha='center', va='center',
fontsize=9, fontweight='bold')
# Draw connections to monitoring
ax.annotate('', xy=(x, 0.15), xytext=(x, y-0.08),
arrowprops=dict(arrowstyle='->',
connectionstyle="arc3,rad=-0.2",
alpha=0.5))
# Draw data flow arrows between components
for i in range(len(system_layers)-1):
x1, y1 = system_layers[i][1], system_layers[i][2] - 0.08
x2, y2 = system_layers[i+1][1] - 0.08, system_layers[i+1][2]
ax.annotate('', xy=(x2, y2), xytext=(x1, y1),
arrowprops=dict(arrowstyle='->',
shrinkA=10, shrinkB=10,
connectionstyle="arc3,rad=0.2",
alpha=0.7, linewidth=2))
# Add feedback loop
ax.annotate('', xy=(0.1, 0.82), xytext=(0.9, 0.82),
arrowprops=dict(arrowstyle='->',
connectionstyle="arc3,rad=-0.5",
alpha=0.5, linestyle='--',
color='red'))
ax.text(0.5, 0.85, 'Feedback Loop\n(Policy updates)',
ha='center', va='center', color='red', fontsize=9)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_title('Production RL System Architecture',
fontsize=14, fontweight='bold', pad=20)
plt.show()
print("\n" + "="*80)
print("SYSTEM COMPONENTS DETAIL")
print("="*80)
for component, subcomponents in self.components.items():
print(f"\n{component}:")
for sub in subcomponents:
print(f" • {sub}")
def demonstrate_mlops_pipeline(self):
"""Demonstrate MLOps pipeline for RL"""
print("\n" + "="*80)
print("MLOPS FOR RL: CONTINUOUS TRAINING & DEPLOYMENT")
print("="*80)
# Create pipeline visualization
fig, axes = plt.subplots(2, 1, figsize=(14, 8))
# Pipeline diagram
ax = axes[0]
ax.axis('off')
pipeline_stages = [
("Data\nCollection", 0.1, 0.5, '#E3F2FD'),
("Data\nValidation", 0.25, 0.5, '#F3E5F5'),
("Feature\nEngineering", 0.4, 0.5, '#E8F5E8'),
("Model\nTraining", 0.55, 0.5, '#FFF3E0'),
("Model\nEvaluation", 0.7, 0.5, '#FFEBEE'),
("Model\nDeployment", 0.85, 0.5, '#E8EAF6')
]
for i, (stage, x, y, color) in enumerate(pipeline_stages):
# Stage box
ax.add_patch(plt.Rectangle((x-0.07, y-0.07), 0.14, 0.14,
facecolor=color, alpha=0.8,
edgecolor='black', linewidth=2))
ax.text(x, y, stage, ha='center', va='center',
fontsize=9, fontweight='bold')
# Stage number
ax.text(x, y+0.1, f'Stage {i+1}',
ha='center', va='center', fontsize=8)
# Connect stages
if i < len(pipeline_stages) - 1:
next_x = pipeline_stages[i+1][1] - 0.07
ax.annotate('', xy=(next_x, y), xytext=(x+0.07, y),
arrowprops=dict(arrowstyle='->', lw=1.5))
# Add trigger sources
triggers = [
("New Data\nAvailable", 0.1, 0.8),
("Model Drift\nDetected", 0.4, 0.8),
("Code\nChanges", 0.55, 0.8),
("Scheduled\nRetraining", 0.85, 0.8)
]
for trigger, x, y in triggers:
ax.text(x, y, trigger, ha='center', va='center',
fontsize=8, style='italic',
bbox=dict(boxstyle='round', facecolor='yellow', alpha=0.5))
ax.annotate('', xy=(x, y-0.05), xytext=(x, 0.57),
arrowprops=dict(arrowstyle='->',
linestyle=':', alpha=0.7))
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_title('Continuous RL Training Pipeline',
fontsize=12, fontweight='bold')
# Monitoring dashboard
ax = axes[1]
# Simulate monitoring metrics
time_points = np.arange(100)
# Performance metrics
reward_metric = 0.7 + 0.3 * np.sin(time_points/10) + 0.05 * np.random.randn(100)
safety_metric = 0.9 - 0.1 * np.sin(time_points/15) + 0.02 * np.random.randn(100)
efficiency_metric = 0.8 + 0.15 * np.cos(time_points/20) + 0.03 * np.random.randn(100)
# Plot metrics
ax.plot(time_points, reward_metric, 'b-', label='Average Reward', linewidth=2)
ax.plot(time_points, safety_metric, 'g-', label='Safety Score', linewidth=2)
ax.plot(time_points, efficiency_metric, 'r-', label='Efficiency', linewidth=2)
# Add deployment markers
deployment_times = [20, 45, 70]
for deploy_time in deployment_times:
ax.axvline(x=deploy_time, color='purple', linestyle='--', alpha=0.7)
ax.text(deploy_time, 0.95, 'Deploy', rotation=90,
ha='right', va='top', color='purple')
ax.set_xlabel('Time (days)')
ax.set_ylabel('Metric Value')
ax.set_title('RL System Monitoring Dashboard')
ax.legend()
ax.grid(True, alpha=0.3)
ax.set_ylim(0, 1)
plt.tight_layout()
plt.show()
print("\nBEST PRACTICES FOR PRODUCTION RL:")
print("-"*80)
practices = [
("Version Everything", "Code, data, models, hyperparameters"),
("Automate Testing", "Unit tests, integration tests, stress tests"),
("Monitor Continuously", "Performance, data drift, resource usage"),
("Plan for Failures", "Rollback strategies, safe exploration"),
("Document Thoroughly", "System design, decisions, limitations"),
("Start Small", "Pilot projects before full deployment")
]
for practice, details in practices:
print(f"• {practice:25s}: {details}")
def create_final_checklist(self):
"""Create final checklist for RL projects"""
print("\n" + "="*80)
print("RL PROJECT CHECKLIST")
print("="*80)
checklist = {
"Problem Formulation": [
"Clearly define the MDP (states, actions, rewards)",
"Consider partial observability",
"Design appropriate reward function",
"Identify constraints and safety requirements"
],
"Algorithm Selection": [
"Match algorithm to problem characteristics",
"Consider sample efficiency requirements",
"Evaluate computational constraints",
"Plan for exploration strategy"
],
"Implementation": [
"Start with simple baselines",
"Implement proper logging and visualization",
"Use version control from day one",
"Write modular, testable code"
],
"Experimentation": [
"Systematic hyperparameter tuning",
"Statistical significance testing",
"Ablation studies",
"Compare against appropriate baselines"
],
"Deployment": [
"Robustness testing",
"Performance benchmarking",
"Safety validation",
"Monitoring infrastructure"
],
"Ethics & Safety": [
"Consider potential misuse",
"Plan for failure modes",
"Document limitations",
"Establish governance procedures"
]
}
for category, items in checklist.items():
print(f"\n{category.upper()}:")
for item in items:
print(f" ☐ {item}")
# Create and demonstrate complete system
system = ProductionRLSystem()
system.visualize_system_architecture()
system.demonstrate_mlops_pipeline()
system.create_final_checklist()
print("\n" + "="*80)
print("THE RL JOURNEY: FROM THEORY TO PRACTICE")
print("="*80)
journey_stages = [
("Chapter 1-3: Foundations", "MDPs, Dynamic Programming, Monte Carlo"),
("Chapter 4-5: Value-Based Methods", "Q-learning, DQN, Deep RL"),
("Chapter 6: Policy-Based Methods", "REINFORCE, Policy Gradients"),
("Chapter 7: Actor-Critic Methods", "A2C, A3C, PPO, SAC"),
("Chapter 8: Advanced Topics", "Exploration, Transfer Learning, Hierarchical RL"),
("Chapter 9: Deep Dive", "Transformers, Attention, Advanced Architectures"),
("Chapter 10: The Frontier", "Modern Algorithms, Real-World Challenges, Future")
]
print("\nYour learning journey through this book:")
for stage, description in journey_stages:
print(f"• {stage:30s}: {description}")
print("\n" + "="*80)
print("FINAL WORDS OF ENCOURAGEMENT")
print("="*80)
final_thoughts = [
"Reinforcement Learning is one of the most exciting fields in AI.",
"You're learning a technology that's shaping the future.",
"The challenges are significant, but so are the opportunities.",
"Every expert was once where you are now.",
"Your unique perspective could lead to the next breakthrough.",
"Go build, experiment, learn, and contribute!"
]
for thought in final_thoughts:
print(f"• {thought}")
print("\n" + "="*80)
print("🚀 THE ADVENTURE CONTINUES... 🚀")
print("="*80)
# Create complete RL system
print("\n" + "="*80)
print("BUILDING COMPLETE RL SYSTEMS")
print("="*80)
complete_system = CompleteRLSystem()
print("\n" + "="*80)
print("CHAPTER 10 COMPLETE!")
print("="*80)
print("\nCongratulations on completing this comprehensive journey through")
print("Reinforcement Learning! You now have:")
print("\n• A deep understanding of RL fundamentals")
print("• Practical experience with modern algorithms")
print("• Awareness of real-world challenges")
print("• Vision for the future of the field")
print("\nThe frontier awaits your contributions!")
print("="*80)This is the complete Chapter 10. It's structured as:
- 10.1 Modern RL Algorithms - Overview of cutting-edge algorithms
- 10.2 Real-World Challenges - Practical issues and solutions
- 10.3 Multi-Agent RL - Systems with multiple interacting agents
- 10.4 Meta-RL - Learning to learn quickly
- 10.5 RLHF - How modern AI systems are trained
- 10.6 Future of RL - Where the field is heading
- 10.7 Complete Systems - Building production-ready RL systems
Each section includes:
- Theoretical explanations
- Visualizations
- Code demonstrations
- Practical insights
- Key takeaways
The chapter builds from fundamentals to frontier research, ending with practical advice for building real-world systems.