Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions python/cli/diagnose/diagnose.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def generateReport(args):
'magnetometer': {"v": [], "t": [], "td": []},
'barometer': {"v": [], "t": [], "td": []},
'gnss': {"v": [], "t": [], "td": []},
'cpu': {"v": [], "t": []},
'cpu': {"v": [], "t": [], "td": [], "processes": {}},
'cameras': {}
}

Expand Down Expand Up @@ -120,8 +120,19 @@ def addMeasurement(type, t, v):
if "features" in f: cameras[ind]["features"].append(len(f["features"]))
cameras[ind]["t"].append(t)
elif metrics is not None and 'cpu' in metrics:
data["cpu"]["t"].append(t)
data["cpu"]["v"].append(metrics['cpu'].get('systemTotalUsagePercent', 0))
addMeasurement("cpu", t, metrics['cpu'].get('systemTotalUsagePercent', 0))
usedProcessNames = {} # Track duplicate process names
for process in metrics['cpu'].get('processes', []):
name = process.get('name')
if not name: continue

count = usedProcessNames.get(name, 0)
usedProcessNames[name] = count + 1
uniqueName = f"{name} {count + 1}" if count else name

processData = data['cpu']["processes"].setdefault(uniqueName, {"v": [], "t": []})
processData['v'].append(process['usagePercent'])
processData['t'].append(t)

if nSkipped > 0: print(f'Skipped {nSkipped} lines')

Expand All @@ -131,7 +142,7 @@ def addMeasurement(type, t, v):
diagnoseMagnetometer(data, output)
diagnoseBarometer(data, output)
diagnoseGNSS(data, output)
diagnoseCpu(data, output)
diagnoseCPU(data, output)

if os.path.dirname(args.output_html):
os.makedirs(os.path.dirname(args.output_html), exist_ok=True)
Expand Down
2 changes: 1 addition & 1 deletion python/cli/diagnose/html.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ def generateHtml(output, outputHtml):
camera["frequency"],
camera["count"])))

SENSOR_NAMES = ["accelerometer", "gyroscope", "magnetometer", "barometer", "GNSS"]
SENSOR_NAMES = ["accelerometer", "gyroscope", "magnetometer", "barometer", "GNSS", "CPU"]
for sensor in SENSOR_NAMES:
if sensor not in output: continue
kvPairs.append((
Expand Down
67 changes: 60 additions & 7 deletions python/cli/diagnose/sensors.py
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ def highpass(signal, fs, cutoff, order=3):
plt.ylabel(measurementUnit)

fig.suptitle(f"Preview of {sensorName} signal noise (mean={noiseScale:.1f}{measurementUnit}, threshold={noiseThreshold:.1f}{measurementUnit})")
fig.tight_layout(rect=[0, 0.03, 1, 0.95])
fig.tight_layout()
self.images.append(base64(fig))

if noiseScale > noiseThreshold:
Expand Down Expand Up @@ -365,6 +365,50 @@ def analyzeAccelerometerSignalHasGravity(self, signal):
"This suggests the signal may be missing gravitational acceleration."
)

def analyzeCpuUsage(self, signal, timestamps, processes):
CPU_USAGE_THRESHOLD = 90.0

mean = np.mean(signal)
p95 = np.percentile(signal, 95)
p99 = np.percentile(signal, 99)

if mean > CPU_USAGE_THRESHOLD:
self.__addIssue(DiagnosisLevel.WARNING,
f"Average CPU usage {mean:.1f}% is above the threshold ({CPU_USAGE_THRESHOLD:.1f}%)."
)
elif p95 > CPU_USAGE_THRESHOLD:
self.__addIssue(DiagnosisLevel.WARNING,
f"95th percentile CPU usage {p95:.1f}% is above the threshold ({CPU_USAGE_THRESHOLD:.1f}%)."
)
elif p99 > CPU_USAGE_THRESHOLD:
self.__addIssue(DiagnosisLevel.WARNING,
f"99th percentile CPU usage {p99:.1f}% is above the threshold ({CPU_USAGE_THRESHOLD:.1f}%)."
)

import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(8, 6))

ax.plot(timestamps, signal, label="System total", linestyle='-')
ax.set_title("CPU usage")
ax.set_ylabel("CPU usage (%)")
ax.set_xlabel("Time (s)")

legend = ['System total']
ylim = 100
for name, data in processes.items():
if len(data['v']) == 0: continue
ax.plot(data['t'], data['v'], label=name, linestyle='--')
legend.append(name)
ylim = max(ylim, np.max(data['v']) * 1.1)

ax.set_ylim(0, ylim)

leg = ax.legend(legend, fontsize='large', markerscale=10)
for line in leg.get_lines(): line.set_linewidth(2)

fig.tight_layout()
self.images.append(base64(fig))

def serializeIssues(self):
self.issues = sorted(self.issues, key=lambda x: x[0], reverse=True)
return [{
Expand Down Expand Up @@ -705,13 +749,22 @@ def diagnoseGNSS(data, output):
if status.diagnosis == DiagnosisLevel.ERROR:
output["passed"] = False

def diagnoseCpu(data, output):
data = data["cpu"]
timestamps = np.array(data["t"])
values = data["v"]
def diagnoseCPU(data, output):
sensor = data["cpu"]
timestamps = np.array(sensor["t"])
deltaTimes = np.array(sensor["td"])
signal = np.array(sensor['v'])
processes = sensor["processes"]

if len(timestamps) == 0: return

output["cpu"] = {
"image": plotFrame(timestamps, values, "CPU system load (%)", ymin=0, ymax=100)
status = Status()
status.analyzeCpuUsage(signal, timestamps, processes)

output["CPU"] = {
"diagnosis": status.diagnosis.toString(),
"issues": status.serializeIssues(),
"frequency": computeSamplingRate(deltaTimes),
"count": len(timestamps),
"images": status.images
}