-
Notifications
You must be signed in to change notification settings - Fork 0
/
ppo_sim.cpp
107 lines (90 loc) · 3.51 KB
/
ppo_sim.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
#include <iomanip>
#include <iostream>
#include <memory>
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>
#include "Environment/Environment.h"
#include "PPOAgent.hpp"
constexpr int16_t kNumAgents{15};
int32_t pickResetPosition(const Environment &env, const Agent *agent)
{
return GetRandomValue(0, static_cast<int32_t>(env.race_track_->track_data_points_.x_m.size()) - 1);
}
int main(int argc, char **argv)
{
if (argc != 2)
{
std::cerr << "Provide a file path for trace track csv file\n";
return -1;
}
Environment env(argv[1]);
std::vector<rl::PPOAgent> agents;
agents.reserve(kNumAgents);
const float start_pos_x{env.race_track_->track_data_points_.x_m[RaceTrack::kStartingIdx]};
const float start_pos_y{env.race_track_->track_data_points_.y_m[RaceTrack::kStartingIdx]};
for (int16_t i{0}; i < kNumAgents; i++)
{
agents.emplace_back(rl::PPOAgent({start_pos_x, start_pos_y},
env.race_track_->headings_[RaceTrack::kStartingIdx],
i,
env.race_track_->findNearestTrackIndexBruteForce({start_pos_x, start_pos_y}),
static_cast<int64_t>(env.race_track_->track_data_points_.x_m.size())));
env.setAgent(&agents.back());
}
uint32_t episode_idx{0};
int32_t reset_idx{RaceTrack::kStartingIdx};
env.visualizer_->user_draw_callback_ = [&episode_idx, &agents]()
{
char buffer[30];
snprintf(buffer, sizeof(buffer), "Episode: %d", episode_idx);
DrawText(buffer, kScreenWidth - 150, 40, 20, YELLOW);
};
bool all_done{false};
while (true)
{
std::cout << "------------ EPISODE " << episode_idx << " DONE ---------------" << std::endl;
all_done = false;
for (auto &agent : agents)
{
agent.reset(
{env.race_track_->track_data_points_.x_m[reset_idx],
env.race_track_->track_data_points_.y_m[reset_idx]},
env.race_track_->headings_[reset_idx],
env.race_track_->findNearestTrackIndexBruteForce({env.race_track_->track_data_points_.x_m[reset_idx],
env.race_track_->track_data_points_.y_m[reset_idx]}));
}
// need to get an initial observation for the intial action, after reset
env.step();
while (!all_done)
{
for (auto &agent : agents)
{
agent.current_state_tensor_ = agent.stateToTensor();
agent.experience_buffer_.saved_states.push_back(agent.current_state_tensor_.detach());
agent.updateAction();
agent.experience_buffer_.saved_actions.push_back(agent.getCurrentAction());
}
env.step();
all_done = true;
for (auto &agent : agents)
{
agent.experience_buffer_.saved_rewards.push_back(1.0F);
if (!agent.crashed_)
{
// agent.experience_buffer_.saved_rewards.push_back(1.0F);
all_done = false;
}
// else
// {
// agent.experience_buffer_.saved_rewards.push_back(-5.F);
// }
}
}
rl::PPOAgent::updatePolicy();
episode_idx++;
reset_idx = pickResetPosition(env, &agents.front());
}
return 0;
}