Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enhance log analysis #32

Open
wants to merge 30 commits into
base: master
from
Open
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
9796ab9
A bit of a cleanup in log-analysis file
Jun 22, 2019
4f9fcd4
Add normalize_reward to log_analysis, a bit of cleanup, added colour …
Jun 22, 2019
e71484a
Add track_utils file
Jun 22, 2019
861a998
Avoid downloading of an already existent file, but allow forcing it
Jun 24, 2019
2f0e773
Improve plotting of grid world for evaluation (to work with negative …
Jun 30, 2019
546ad3c
Move plotting of top laps from the notebook
Jun 30, 2019
8d96466
Add functions to aggregate training data
Jun 30, 2019
3ccd777
Add functions to load and analyse evaluations in bulk
Jun 30, 2019
62b356b
readme, gitignore
Jun 30, 2019
5bb33a2
Add evaluation analysis
Jul 1, 2019
df39701
small fixes, store timestamp as decimal to make it readable
Jul 1, 2019
209ac8e
add code for reward recalculation
Jul 2, 2019
9730bb6
Add reward calculation, fix reward, Provide a Training_analysis.ipynb
Jul 2, 2019
6005efa
Make episodes per iteration configurable when parsing logs
Jul 2, 2019
8545d7f
Add action breakdown and small improvements all around
Jul 2, 2019
ba2fd9a
Add evaluation analysis notebook
Jul 2, 2019
3e86859
Improve the readme
Jul 2, 2019
07cc2a2
Improve the readme
Jul 2, 2019
b62a2a3
Add graphs per starting point to aggregates
Jul 12, 2019
dc5a48a
Set x axis in progress of buckets to (0,100)
Jul 12, 2019
e974b51
Add minimum and maximum complete times per iteration
Jul 12, 2019
79b1851
New York tracks for analysis
Jul 13, 2019
5e0c209
Fix minimum and maximum time plot
Jul 13, 2019
92f276f
Small updates and cleanups, reruns of updated functions
Jul 13, 2019
46e15f5
Clean up aggregate graphs for evaluation and fix timestap issue in no…
Jul 17, 2019
5f5399c
Preserve shape of top laps graphs
Jul 17, 2019
bd61533
Speed up analysis of training progress and reorganise plotting in tra…
Jul 17, 2019
cc95464
Speed up plotting of evaluation laps
Jul 18, 2019
18deb7f
Updated notebooks to fixes, fine-tune log_analysis, fix some memory i…
Jul 20, 2019
4c69b6f
Added China training track npy
breadcentric Aug 3, 2019
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

Speed up plotting of evaluation laps

  • Loading branch information...
Tomasz Ptak
Tomasz Ptak committed Jul 18, 2019
commit cc954641f7f615fd03f41b091f6072add8013098
@@ -164,8 +164,7 @@
"\n",
"There is also `older_than` parameter so you can choose to fetch all logs from a given month and compare them against each other.\n",
"\n",
"As mentioned, this method always fetches a list of log streams and then downloads only ones that haven't been downloaded just yet. You can therefore use it to fetch that list and load all the files from the path provided."
"\n",
"As mentioned, this method always fetches a list of log streams and then downloads only ones that haven't been downloaded just yet. You can therefore use it to fetch that list and load all the files from the path provided.\n",
"It's good to keep things organised: group your files into folders to not lose track where they came from. Replace `SELECT_YOUR_FOLDER` with a path matching your preference."
]
},
@@ -1048,7 +1047,7 @@
}
],
"source": [
"la.analyse_multiple_race_evaluations(logs, l_inner_border, l_outer_border, min_distance_to_plot=21)"
"la.plot_evaluations(bulk, l_inner_border, l_outer_border)"
]
},
{
@@ -15,7 +15,6 @@
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""

import math
from datetime import datetime
from decimal import *

@@ -24,7 +23,6 @@
import pandas as pd
from matplotlib.collections import PatchCollection
from matplotlib.patches import Rectangle
from shapely.geometry import Point, Polygon
from shapely.geometry.polygon import LineString
from sklearn.preprocessing import MinMaxScaler

@@ -236,98 +234,64 @@ def plot_top_laps(sorted_idx, episode_map, center_line, inner_border,
return fig


def plot_grid_world(episode_df, inner, outer, scale=10.0, plot=True,
log_tuple=None, min_distance_to_plot=None,
graphed_value='throttle'):
def plot_evaluations(evaluations, inner, outer, graphed_value='throttle'):
streams = evaluations.groupby('stream')

for name, stream in streams:
fig, axes = plt.subplots(2, 3, figsize=(20, 10))
fig.tight_layout(pad=0.4, w_pad=0.5, h_pad=7.0)

for id, episode in stream.groupby('episode'):
plot_grid_world(episode, inner, outer, graphed_value, ax=axes[int(id / 3), id % 3])

fig.show()


def plot_grid_world(episode_df, inner, outer, graphed_value='throttle', min_progress=None, ax=None):
"""
plot a scaled version of lap, along with throttle taken a each position
"""
stats = []
outer = [(val[0] / scale, val[1] / scale) for val in outer]
inner = [(val[0] / scale, val[1] / scale) for val in inner]

max_x = int(np.max([val[0] for val in outer]))
max_y = int(np.max([val[1] for val in outer]))
min_x = min(int(np.min([val[0] for val in outer])), 0)
min_y = min(int(np.min([val[1] for val in outer])), 0)
episode_df.loc[:, 'distance_diff'] = ((episode_df['x'].shift(1) - episode_df['x']) ** 2 + (
episode_df['y'].shift(1) - episode_df['y']) ** 2) ** 0.5

outer = [(val[0] - min_x, val[1] - min_y) for val in outer]
inner = [(val[0] - min_x, val[1] - min_y) for val in inner]
distance = np.nansum(episode_df['distance_diff']) / 100
lap_time = np.ptp(episode_df['timestamp'].astype(float))
velocity = 0.01 * distance / lap_time
average_throttle = np.nanmean(episode_df['throttle'])
progress = np.nanmax(episode_df['progress'])

grid = np.zeros((max_x + 1 - min_x, max_y + 1 - min_y))
if not min_progress or progress > min_progress:

# create shapely ring for outter and inner
outer_polygon = Polygon(outer)
inner_polygon = Polygon(inner)
distance_lap_time = 'Distance, progress, lap time = %.2f (meters), %.2f %%, %.2f (sec)' % (
distance, progress, lap_time)
throttle_velocity = 'Average throttle, velocity = %.2f (Gazebo), %.2f (meters/sec)' % (
average_throttle, velocity)

print('Outer polygon length = %.2f (meters)' % (
outer_polygon.length / scale))
print('Inner polygon length = %.2f (meters)' % (
inner_polygon.length / scale))
fig = None
if ax is None:
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)

dist = 0.0
for ii in range(1, len(episode_df)):
dist += math.sqrt(
(episode_df['x'].iloc[ii] - episode_df['x'].iloc[ii - 1]) ** 2 + (
episode_df['y'].iloc[ii] - episode_df['y'].iloc[
ii - 1]) ** 2)
dist /= 100.0
ax.set_facecolor('midnightblue')

t0 = datetime.fromtimestamp(float(episode_df['timestamp'].iloc[0]))
t1 = datetime.fromtimestamp(
float(episode_df['timestamp'].iloc[len(episode_df) - 1]))
line = LineString(inner)
plot_coords(ax, line)
plot_line(ax, line)

line = LineString(outer)
plot_coords(ax, line)
plot_line(ax, line)

lap_time = (t1 - t0).total_seconds()
episode_df.plot.scatter('x', 'y', ax=ax, s=3, c=graphed_value, cmap=plt.get_cmap('plasma'))

average_throttle = np.nanmean(episode_df['throttle'])
max_throttle = np.nanmax(episode_df['throttle'])
min_throttle = np.nanmin(episode_df['throttle'])
velocity = dist / lap_time

distance_lap_time = 'Distance, lap time = %.2f (meters), %.2f (sec)' % (
dist, lap_time)
print(distance_lap_time)
throttle_velocity = 'Average throttle, velocity = %.2f (Gazebo), %.2f (meters/sec)' % (
average_throttle, velocity)
print(throttle_velocity)

stats.append((dist, lap_time, velocity, average_throttle, min_throttle,
max_throttle))

if plot == True and (not min_distance_to_plot or dist > min_distance_to_plot):
for y in range(max_y - min_y):
for x in range(max_x - min_x):
point = Point((x, y))

# this is the track
if (not inner_polygon.contains(point)) and (
outer_polygon.contains(point)):
grid[x][y] = -1.0

# find df slice that fits into this
df_slice = episode_df[
(episode_df['x'] >= (x + min_x - 1) * scale) & (
episode_df['x'] < (x + min_x) * scale) & \
(episode_df['y'] >= (y + min_y - 1) * scale) & (
episode_df['y'] < (y + min_y) * scale)]

if len(df_slice) > 0:
# average_throttle = np.nanmean(df_slice['throttle'])
grid[x][y] = np.nanmean(df_slice[graphed_value])

fig = plt.figure(figsize=(12, 16))
imgplot = plt.imshow(grid)
subtitle = ''
if log_tuple:
subtitle = '\n%s\n%s\n%s\n%s' % (
log_tuple[1], datetime.fromtimestamp(log_tuple[2] / 1000.0),
distance_lap_time, throttle_velocity)
plt.colorbar(orientation='horizontal')
plt.title('Lap time (sec) = %.3f%s' % (lap_time, subtitle))
plt.show()
plt.clf()

return lap_time, average_throttle, stats
subtitle = 'Stream: %s, %s\n%s\n%s' % (
episode_df['stream'].iloc[0], datetime.fromtimestamp(episode_df['timestamp'].iloc[0]),
distance_lap_time, throttle_velocity)
ax.set_title(subtitle)

if fig:
fig.show()


def simulation_agg(panda, firstgroup='iteration', add_timestamp=False, is_eval=False):
@@ -481,31 +445,26 @@ def load_eval_logs(logs):


def analyse_single_evaluation(log_file, inner_border, outer_border, episodes=5,
log_tuple=None, min_distance_to_plot=None):
print("###############################################################")
print(log_file)
min_progress=None):
eval_df = load_eval_data(log_file)

for e in range(episodes):
print("\nEpisode #%s " % e)
episode_df = eval_df[eval_df['episode'] == e]
plot_grid_world(episode_df, inner_border, outer_border, scale=5.0,
log_tuple=log_tuple, min_distance_to_plot=min_distance_to_plot)
plot_grid_world(episode_df, inner_border, outer_border, min_progress=min_progress)


def analyse_multiple_race_evaluations(logs, inner_border, outer_border, min_distance_to_plot=None):
def analyse_multiple_race_evaluations(logs, inner_border, outer_border, min_progress=None):
for log in logs:
analyse_single_evaluation(log[0], inner_border, outer_border,
log_tuple=log, min_distance_to_plot=min_distance_to_plot)
analyse_single_evaluation(log[0], inner_border, outer_border, min_progress=min_progress)


def download_and_analyse_multiple_race_evaluations(log_folder, l_inner_border, l_outer_border, not_older_than=None,
older_than=None,
log_group='/aws/deepracer/leaderboard/SimulationJobs',
min_distance_to_plot=None):
min_progress=None):
logs = cw.download_all_logs("%s/deepracer-eval-" % log_folder, log_group, not_older_than, older_than)

analyse_multiple_race_evaluations(logs, l_inner_border, l_outer_border, min_distance_to_plot=min_distance_to_plot)
analyse_multiple_race_evaluations(logs, l_inner_border, l_outer_border, min_progress=min_progress)


def df_to_params(df_row, waypoints):
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.