diff --git a/.claude/scripts/io-performance-analyzer/Dockerfile b/.claude/scripts/io-performance-analyzer/Dockerfile new file mode 100644 index 0000000000..83b85bff2c --- /dev/null +++ b/.claude/scripts/io-performance-analyzer/Dockerfile @@ -0,0 +1,10 @@ +FROM registry.fedoraproject.org/fedora:43 + +RUN dnf install -y pcp pcp-system-tools pcp-export-pcp2json python3 python3-pip && \ + dnf clean all + +RUN pip3 install matplotlib gsutil + +WORKDIR /data + +ENTRYPOINT ["/bin/bash"] diff --git a/.claude/scripts/io-performance-analyzer/README.md b/.claude/scripts/io-performance-analyzer/README.md new file mode 100644 index 0000000000..d840e2fa8b --- /dev/null +++ b/.claude/scripts/io-performance-analyzer/README.md @@ -0,0 +1,117 @@ +# Disk I/O Performance Graphing for MicroShift CI + +Tool for visualizing disk I/O activity from MicroShift CI job runs, primarily +aimed at investigating etcd performance issues. + +## Background + +MicroShift uses an embedded etcd instance that is highly sensitive to disk I/O +latency. Symptoms such as slow leader elections, "apply request took too long" +warnings, or WAL sync delays often correlate with disk I/O spikes on the CI +node. CI jobs collect [Performance Co-Pilot (PCP)](https://pcp.io/documentation.html) +archives via the `pcp-zeroconf` package throughout the test run, capturing +system-wide performance metrics at high resolution. + +This tool processes those PCP archives and produces a time-series graph of +**Disk Read OPS**, **Disk Write OPS**, and **Disk Await** (`disk.dev.await`) +at 15-second intervals, making it straightforward to correlate etcd issues +with underlying disk activity. + +**Disk Await** is the average time (in milliseconds) that I/O requests spend +waiting to be serviced by the device, including queue time and actual service +time. It is the single most useful metric for diagnosing etcd I/O problems +because etcd requires low-latency `fdatasync` calls on its WAL and snap files. +When await rises above ~10 ms, etcd heartbeats can be missed and leader +elections may be triggered. The tool reports the **max await across all block +devices** at each sample point so that the worst-case device is always visible. + +## Obtaining PCP Data from CI + +PCP archives are stored with the CI artifacts under: + +```text +artifacts//openshift-microshift-infra-pmlogs/artifacts// +``` + +Download them using `gsutil`: + +```bash +mkdir -p ~/pmlogs && cd ~/pmlogs +python3 -m venv . +./bin/python3 -m pip install gsutil +./bin/gsutil -m cp -r gs:// ~/pmlogs/ +``` + +The directory should contain files like `yyyymmdd.hh.mm.{0,index,meta}` and a +`Latest` folio file. + +## Prerequisites + +- `podman` (used to build and run the analysis container) + +No other local dependencies are required. The container image includes PCP +tools, Python 3, and matplotlib. + +## Usage + +```bash +./run_io_graph.sh [--timezone TZ] [--output-dir DIR] [pcp-data-dir] +``` + +| Option | Description | Default | +|---|---|---| +| `--timezone TZ` | IANA timezone for timestamps | `UTC` | +| `--output-dir DIR` | Directory for output files | Script directory | +| `pcp-data-dir` | Directory with PCP archive files | Auto-detected | + +### Examples + +```bash +# Auto-detect PCP data, default timezone (UTC) +./run_io_graph.sh + +# Specify data directory and timezone +./run_io_graph.sh --timezone US/Eastern ./path/to/pcp-data + +# Custom output directory +./run_io_graph.sh --output-dir /tmp/results --timezone UTC ./path/to/pcp-data +``` + +## Output + +| File | Description | +|---|---| +| `io_data.json` | Extracted data with arrays: `timestamps`, `bi` (reads/s), `bo` (writes/s), `await` (disk await ms) | +| `disk_io_performance.png` | Time-series chart with Read OPS (blue), Write OPS (red), and Disk Await (green, dashed, right Y-axis) | + +### Sample Graph + +![Disk I/O Performance](disk_io_performance.png) + +## How to Read the Graph + +When investigating etcd performance problems, look for: + +- **Disk Await spikes** (green, dashed, right Y-axis) are the primary + indicator of I/O latency problems. etcd requires `fdatasync` to complete + within its heartbeat interval (default 100 ms). Await values above ~10 ms + indicate the disk is under pressure; sustained values above ~50 ms almost + always correlate with etcd warnings such as "slow fdatasync", "apply request + took too long", or missed heartbeats leading to leader elections. +- **Write spikes** (red) coinciding with etcd "slow fdatasync" or WAL warnings + in MicroShift journal logs. Sustained write activity above the baseline + suggests I/O contention from concurrent workloads or test operations. +- **Read spikes** (blue) during test setup or image pulls that may starve etcd + of I/O bandwidth. +- **Correlation with timestamps** from etcd log entries. Convert etcd log + timestamps to the timezone used in the graph to align events. + +## Files + +| File | Purpose | +|---|---| +| `run_io_graph.sh` | Orchestrator: builds container, runs extraction and plotting | +| `Dockerfile` | Container image with PCP tools, Python 3, and matplotlib | +| `extract_io.sh` | Runs `pcp2json` on PCP archives, extracts all metrics in one pass | +| `parse_pcp.py` | Parses pcp2json output, aggregates per-device instances (sum read/write, max await) | +| `plot_io.py` | Generates the PNG chart from JSON data using matplotlib | diff --git a/.claude/scripts/io-performance-analyzer/disk_io_performance.png b/.claude/scripts/io-performance-analyzer/disk_io_performance.png new file mode 100644 index 0000000000..3cd2695c2e Binary files /dev/null and b/.claude/scripts/io-performance-analyzer/disk_io_performance.png differ diff --git a/.claude/scripts/io-performance-analyzer/extract_io.sh b/.claude/scripts/io-performance-analyzer/extract_io.sh new file mode 100755 index 0000000000..c4cab8217f --- /dev/null +++ b/.claude/scripts/io-performance-analyzer/extract_io.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Extract disk IO data from PCP archive using pcp2json with 15-second intervals. +# Outputs JSON with arrays: timestamps, bi (reads/s), bo (writes/s), await (disk await ms) +# +# Usage: ./extract_io.sh [output-json] [timezone] + +set -euo pipefail + +DATA_DIR="${1:?Usage: $0 [output-json] [timezone]}" +OUTPUT="${2:-/data/io_data.json}" +TIMEZONE="${3:-UTC}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "Using archive directory: ${DATA_DIR}" + +# Extract all metrics in a single pcp2json call +TMPFILE=$(mktemp) +trap 'rm -f "${TMPFILE}"' EXIT + +(cd "${DATA_DIR}" && pcp2json -a . -t 15sec \ + disk.dev.read disk.dev.write disk.dev.await) \ + > "${TMPFILE}" 2>/dev/null || true + +# Parse pcp2json output into clean plot-ready JSON +python3 "${SCRIPT_DIR}/parse_pcp.py" --timezone "${TIMEZONE}" \ + "${TMPFILE}" "${OUTPUT}" + +echo "Wrote $(python3 -c "import json; d=json.load(open('${OUTPUT}')); print(len(d['timestamps']))" 2>/dev/null || echo 0) data points to ${OUTPUT}" diff --git a/.claude/scripts/io-performance-analyzer/io_data.json b/.claude/scripts/io-performance-analyzer/io_data.json new file mode 100644 index 0000000000..41638bbc4c --- /dev/null +++ b/.claude/scripts/io-performance-analyzer/io_data.json @@ -0,0 +1,3114 @@ +{ + "timestamps": [ + "2026-02-15 03:12:30", + "2026-02-15 03:12:40", + "2026-02-15 03:12:50", + "2026-02-15 03:13:00", + "2026-02-15 03:13:10", + "2026-02-15 03:13:20", + "2026-02-15 03:13:30", + "2026-02-15 03:13:40", + "2026-02-15 03:13:50", + "2026-02-15 03:14:00", + "2026-02-15 03:14:10", + "2026-02-15 03:14:20", + "2026-02-15 03:14:30", + "2026-02-15 03:14:40", + "2026-02-15 03:14:50", + "2026-02-15 03:15:00", + "2026-02-15 03:15:10", + "2026-02-15 03:15:20", + "2026-02-15 03:15:30", + "2026-02-15 03:15:40", + "2026-02-15 03:15:50", + "2026-02-15 03:16:00", + "2026-02-15 03:16:10", + "2026-02-15 03:16:20", + "2026-02-15 03:16:30", + "2026-02-15 03:16:40", + "2026-02-15 03:16:50", + "2026-02-15 03:17:00", + "2026-02-15 03:17:10", + "2026-02-15 03:17:20", + "2026-02-15 03:17:30", + "2026-02-15 03:17:40", + "2026-02-15 03:17:50", + "2026-02-15 03:18:00", + "2026-02-15 03:18:10", + "2026-02-15 03:18:20", + "2026-02-15 03:18:30", + "2026-02-15 03:18:40", + "2026-02-15 03:18:50", + "2026-02-15 03:19:00", + "2026-02-15 03:19:10", + "2026-02-15 03:19:20", + "2026-02-15 03:19:30", + "2026-02-15 03:19:40", + "2026-02-15 03:19:50", + "2026-02-15 03:20:00", + "2026-02-15 03:20:10", + "2026-02-15 03:20:20", + "2026-02-15 03:20:30", + "2026-02-15 03:20:40", + "2026-02-15 03:20:50", + "2026-02-15 03:21:00", + "2026-02-15 03:21:10", + "2026-02-15 03:21:20", + "2026-02-15 03:21:30", + "2026-02-15 03:21:40", + "2026-02-15 03:21:50", + "2026-02-15 03:22:00", + "2026-02-15 03:22:10", + "2026-02-15 03:22:20", + "2026-02-15 03:22:30", + "2026-02-15 03:22:40", + "2026-02-15 03:22:50", + "2026-02-15 03:23:00", + "2026-02-15 03:23:10", + "2026-02-15 03:23:20", + "2026-02-15 03:23:30", + "2026-02-15 03:23:40", + "2026-02-15 03:23:50", + "2026-02-15 03:24:00", + "2026-02-15 03:24:10", + "2026-02-15 03:24:20", + "2026-02-15 03:24:30", + "2026-02-15 03:24:40", + "2026-02-15 03:24:50", + "2026-02-15 03:25:00", + "2026-02-15 03:25:10", + "2026-02-15 03:25:20", + "2026-02-15 03:25:30", + "2026-02-15 03:25:40", + "2026-02-15 03:25:50", + "2026-02-15 03:26:00", + "2026-02-15 03:26:10", + "2026-02-15 03:26:20", + "2026-02-15 03:26:30", + "2026-02-15 03:26:40", + "2026-02-15 03:26:50", + "2026-02-15 03:27:00", + "2026-02-15 03:27:10", + "2026-02-15 03:27:20", + "2026-02-15 03:27:30", + "2026-02-15 03:27:40", + "2026-02-15 03:27:50", + "2026-02-15 03:28:00", + "2026-02-15 03:28:10", + "2026-02-15 03:28:20", + "2026-02-15 03:28:30", + "2026-02-15 03:28:40", + "2026-02-15 03:28:50", + "2026-02-15 03:29:00", + "2026-02-15 03:29:10", + "2026-02-15 03:29:20", + "2026-02-15 03:29:30", + "2026-02-15 03:29:40", + "2026-02-15 03:29:50", + "2026-02-15 03:30:00", + "2026-02-15 03:30:10", + "2026-02-15 03:30:20", + "2026-02-15 03:30:30", + "2026-02-15 03:30:40", + "2026-02-15 03:30:50", + "2026-02-15 03:31:00", + "2026-02-15 03:31:10", + "2026-02-15 03:31:20", + "2026-02-15 03:31:30", + "2026-02-15 03:31:40", + "2026-02-15 03:31:50", + "2026-02-15 03:32:00", + "2026-02-15 03:32:10", + "2026-02-15 03:32:20", + "2026-02-15 03:32:30", + "2026-02-15 03:32:40", + "2026-02-15 03:32:50", + "2026-02-15 03:33:00", + "2026-02-15 03:33:10", + "2026-02-15 03:33:20", + "2026-02-15 03:33:30", + "2026-02-15 03:33:40", + "2026-02-15 03:33:50", + "2026-02-15 03:34:00", + "2026-02-15 03:34:10", + "2026-02-15 03:34:20", + "2026-02-15 03:34:30", + "2026-02-15 03:34:40", + "2026-02-15 03:34:50", + "2026-02-15 03:35:00", + "2026-02-15 03:35:10", + "2026-02-15 03:35:20", + "2026-02-15 03:35:30", + "2026-02-15 03:35:40", + "2026-02-15 03:35:50", + "2026-02-15 03:36:00", + "2026-02-15 03:36:10", + "2026-02-15 03:36:20", + "2026-02-15 03:36:30", + "2026-02-15 03:36:40", + "2026-02-15 03:36:50", + "2026-02-15 03:37:00", + "2026-02-15 03:37:10", + "2026-02-15 03:37:20", + "2026-02-15 03:37:30", + "2026-02-15 03:37:40", + "2026-02-15 03:37:50", + "2026-02-15 03:38:00", + "2026-02-15 03:38:10", + "2026-02-15 03:38:20", + "2026-02-15 03:38:30", + "2026-02-15 03:38:40", + "2026-02-15 03:38:50", + "2026-02-15 03:39:00", + "2026-02-15 03:39:10", + "2026-02-15 03:39:20", + "2026-02-15 03:39:30", + "2026-02-15 03:39:40", + "2026-02-15 03:39:50", + "2026-02-15 03:40:00", + "2026-02-15 03:40:10", + "2026-02-15 03:40:20", + "2026-02-15 03:40:30", + "2026-02-15 03:40:40", + "2026-02-15 03:40:50", + "2026-02-15 03:41:00", + "2026-02-15 03:41:10", + "2026-02-15 03:41:20", + "2026-02-15 03:41:30", + "2026-02-15 03:41:40", + "2026-02-15 03:41:50", + "2026-02-15 03:42:00", + "2026-02-15 03:42:10", + "2026-02-15 03:42:20", + "2026-02-15 03:42:30", + "2026-02-15 03:42:40", + "2026-02-15 03:42:50", + "2026-02-15 03:43:00", + "2026-02-15 03:43:10", + "2026-02-15 03:43:20", + "2026-02-15 03:43:30", + "2026-02-15 03:43:40", + "2026-02-15 03:43:50", + "2026-02-15 03:44:00", + "2026-02-15 03:44:10", + "2026-02-15 03:44:20", + "2026-02-15 03:44:30", + "2026-02-15 03:44:40", + "2026-02-15 03:44:50", + "2026-02-15 03:45:00", + "2026-02-15 03:45:10", + "2026-02-15 03:45:20", + "2026-02-15 03:45:30", + "2026-02-15 03:45:40", + "2026-02-15 03:45:50", + "2026-02-15 03:46:00", + "2026-02-15 03:46:10", + "2026-02-15 03:46:20", + "2026-02-15 03:46:30", + "2026-02-15 03:46:40", + "2026-02-15 03:46:50", + "2026-02-15 03:47:00", + "2026-02-15 03:47:10", + "2026-02-15 03:47:20", + "2026-02-15 03:47:30", + "2026-02-15 03:47:40", + "2026-02-15 03:47:50", + "2026-02-15 03:48:00", + "2026-02-15 03:48:10", + "2026-02-15 03:48:20", + "2026-02-15 03:48:30", + "2026-02-15 03:48:40", + "2026-02-15 03:48:50", + "2026-02-15 03:49:00", + "2026-02-15 03:49:10", + "2026-02-15 03:49:20", + "2026-02-15 03:49:30", + "2026-02-15 03:49:40", + "2026-02-15 03:49:50", + "2026-02-15 03:50:00", + "2026-02-15 03:50:10", + "2026-02-15 03:50:20", + "2026-02-15 03:50:30", + "2026-02-15 03:50:40", + "2026-02-15 03:50:50", + "2026-02-15 03:51:00", + "2026-02-15 03:51:10", + "2026-02-15 03:51:20", + "2026-02-15 03:51:30", + "2026-02-15 03:51:40", + "2026-02-15 03:51:50", + "2026-02-15 03:52:00", + "2026-02-15 03:52:10", + "2026-02-15 03:52:20", + "2026-02-15 03:52:30", + "2026-02-15 03:52:40", + "2026-02-15 03:52:50", + "2026-02-15 03:53:00", + "2026-02-15 03:53:10", + "2026-02-15 03:53:20", + "2026-02-15 03:53:30", + "2026-02-15 03:53:40", + "2026-02-15 03:53:50", + "2026-02-15 03:54:00", + "2026-02-15 03:54:10", + "2026-02-15 03:54:20", + "2026-02-15 03:54:30", + "2026-02-15 03:54:40", + "2026-02-15 03:54:50", + "2026-02-15 03:55:00", + "2026-02-15 03:55:10", + "2026-02-15 03:55:20", + "2026-02-15 03:55:30", + "2026-02-15 03:55:40", + "2026-02-15 03:55:50", + "2026-02-15 03:56:00", + "2026-02-15 03:56:10", + "2026-02-15 03:56:20", + "2026-02-15 03:56:30", + "2026-02-15 03:56:40", + "2026-02-15 03:56:50", + "2026-02-15 03:57:00", + "2026-02-15 03:57:10", + "2026-02-15 03:57:20", + "2026-02-15 03:57:30", + "2026-02-15 03:57:40", + "2026-02-15 03:57:50", + "2026-02-15 03:58:00", + "2026-02-15 03:58:10", + "2026-02-15 03:58:20", + "2026-02-15 03:58:30", + "2026-02-15 03:58:40", + "2026-02-15 03:58:50", + "2026-02-15 03:59:00", + "2026-02-15 03:59:10", + "2026-02-15 03:59:20", + "2026-02-15 03:59:30", + "2026-02-15 03:59:40", + "2026-02-15 03:59:50", + "2026-02-15 04:00:00", + "2026-02-15 04:00:10", + "2026-02-15 04:00:20", + "2026-02-15 04:00:30", + "2026-02-15 04:00:40", + "2026-02-15 04:00:50", + "2026-02-15 04:01:00", + "2026-02-15 04:01:10", + "2026-02-15 04:01:20", + "2026-02-15 04:01:30", + "2026-02-15 04:01:40", + "2026-02-15 04:01:50", + "2026-02-15 04:02:00", + "2026-02-15 04:02:10", + "2026-02-15 04:02:20", + "2026-02-15 04:02:30", + "2026-02-15 04:02:40", + "2026-02-15 04:02:50", + "2026-02-15 04:03:00", + "2026-02-15 04:03:10", + "2026-02-15 04:03:20", + "2026-02-15 04:03:30", + "2026-02-15 04:03:40", + "2026-02-15 04:03:50", + "2026-02-15 04:04:00", + "2026-02-15 04:04:10", + "2026-02-15 04:04:20", + "2026-02-15 04:04:30", + "2026-02-15 04:04:40", + "2026-02-15 04:04:50", + "2026-02-15 04:05:00", + "2026-02-15 04:05:10", + "2026-02-15 04:05:20", + "2026-02-15 04:05:30", + "2026-02-15 04:05:40", + "2026-02-15 04:05:50", + "2026-02-15 04:06:00", + "2026-02-15 04:06:10", + "2026-02-15 04:06:20", + "2026-02-15 04:06:30", + "2026-02-15 04:06:40", + "2026-02-15 04:06:50", + "2026-02-15 04:07:00", + "2026-02-15 04:07:10", + "2026-02-15 04:07:20", + "2026-02-15 04:07:30", + "2026-02-15 04:07:40", + "2026-02-15 04:07:50", + "2026-02-15 04:08:00", + "2026-02-15 04:08:10", + "2026-02-15 04:08:20", + "2026-02-15 04:08:30", + "2026-02-15 04:08:40", + "2026-02-15 04:08:50", + "2026-02-15 04:09:00", + "2026-02-15 04:09:10", + "2026-02-15 04:09:20", + "2026-02-15 04:09:30", + "2026-02-15 04:09:40", + "2026-02-15 04:09:50", + "2026-02-15 04:10:00", + "2026-02-15 04:10:10", + "2026-02-15 04:10:20", + "2026-02-15 04:10:30", + "2026-02-15 04:10:40", + "2026-02-15 04:10:50", + "2026-02-15 04:11:00", + "2026-02-15 04:11:10", + "2026-02-15 04:11:20", + "2026-02-15 04:11:30", + "2026-02-15 04:11:40", + "2026-02-15 04:11:50", + "2026-02-15 04:12:00", + "2026-02-15 04:12:10", + "2026-02-15 04:12:20", + "2026-02-15 04:12:30", + "2026-02-15 04:12:40", + "2026-02-15 04:12:50", + "2026-02-15 04:13:00", + "2026-02-15 04:13:10", + "2026-02-15 04:13:20", + "2026-02-15 04:13:30", + "2026-02-15 04:13:40", + "2026-02-15 04:13:50", + "2026-02-15 04:14:00", + "2026-02-15 04:14:10", + "2026-02-15 04:14:20", + "2026-02-15 04:14:30", + "2026-02-15 04:14:40", + "2026-02-15 04:14:50", + "2026-02-15 04:15:00", + "2026-02-15 04:15:10", + "2026-02-15 04:15:20", + "2026-02-15 04:15:30", + "2026-02-15 04:15:40", + "2026-02-15 04:15:50", + "2026-02-15 04:16:00", + "2026-02-15 04:16:10", + "2026-02-15 04:16:20", + "2026-02-15 04:16:30", + "2026-02-15 04:16:40", + "2026-02-15 04:16:50", + "2026-02-15 04:17:00", + "2026-02-15 04:17:10", + "2026-02-15 04:17:20", + "2026-02-15 04:17:30", + "2026-02-15 04:17:40", + "2026-02-15 04:17:50", + "2026-02-15 04:18:00", + "2026-02-15 04:18:10", + "2026-02-15 04:18:20", + "2026-02-15 04:18:30", + "2026-02-15 04:18:40", + "2026-02-15 04:18:50", + "2026-02-15 04:19:00", + "2026-02-15 04:19:10", + "2026-02-15 04:19:20", + "2026-02-15 04:19:30", + "2026-02-15 04:19:40", + "2026-02-15 04:19:50", + "2026-02-15 04:20:00", + "2026-02-15 04:20:10", + "2026-02-15 04:20:20", + "2026-02-15 04:20:30", + "2026-02-15 04:20:40", + "2026-02-15 04:20:50", + "2026-02-15 04:21:00", + "2026-02-15 04:21:10", + "2026-02-15 04:21:20", + "2026-02-15 04:21:30", + "2026-02-15 04:21:40", + "2026-02-15 04:21:50", + "2026-02-15 04:22:00", + "2026-02-15 04:22:10", + "2026-02-15 04:22:20", + "2026-02-15 04:22:30", + "2026-02-15 04:22:40", + "2026-02-15 04:22:50", + "2026-02-15 04:23:00", + "2026-02-15 04:23:10", + "2026-02-15 04:23:20", + "2026-02-15 04:23:30", + "2026-02-15 04:23:40", + "2026-02-15 04:23:50", + "2026-02-15 04:24:00", + "2026-02-15 04:24:10", + "2026-02-15 04:24:20", + "2026-02-15 04:24:30", + "2026-02-15 04:24:40", + "2026-02-15 04:24:50", + "2026-02-15 04:25:00", + "2026-02-15 04:25:10", + "2026-02-15 04:25:20", + "2026-02-15 04:25:30", + "2026-02-15 04:25:40", + "2026-02-15 04:25:50", + "2026-02-15 04:26:00", + "2026-02-15 04:26:10", + "2026-02-15 04:26:20", + "2026-02-15 04:26:30", + "2026-02-15 04:26:40", + "2026-02-15 04:26:50", + "2026-02-15 04:27:00", + "2026-02-15 04:27:10", + "2026-02-15 04:27:20", + "2026-02-15 04:27:30", + "2026-02-15 04:27:40", + "2026-02-15 04:27:50", + "2026-02-15 04:28:00", + "2026-02-15 04:28:10", + "2026-02-15 04:28:20", + "2026-02-15 04:28:30", + "2026-02-15 04:28:40", + "2026-02-15 04:28:50", + "2026-02-15 04:29:00", + "2026-02-15 04:29:10", + "2026-02-15 04:29:20", + "2026-02-15 04:29:30", + "2026-02-15 04:29:40", + "2026-02-15 04:29:50", + "2026-02-15 04:30:00", + "2026-02-15 04:30:10", + "2026-02-15 04:30:20", + "2026-02-15 04:30:30", + "2026-02-15 04:30:40", + "2026-02-15 04:30:50", + "2026-02-15 04:31:00", + "2026-02-15 04:31:10", + "2026-02-15 04:31:20", + "2026-02-15 04:31:30", + "2026-02-15 04:31:40", + "2026-02-15 04:31:50", + "2026-02-15 04:32:00", + "2026-02-15 04:32:10", + "2026-02-15 04:32:20", + "2026-02-15 04:32:30", + "2026-02-15 04:32:40", + "2026-02-15 04:32:50", + "2026-02-15 04:33:00", + "2026-02-15 04:33:10", + "2026-02-15 04:33:20", + "2026-02-15 04:33:30", + "2026-02-15 04:33:40", + "2026-02-15 04:33:50", + "2026-02-15 04:34:00", + "2026-02-15 04:34:10", + "2026-02-15 04:34:20", + "2026-02-15 04:34:30", + "2026-02-15 04:34:40", + "2026-02-15 04:34:50", + "2026-02-15 04:35:00", + "2026-02-15 04:35:10", + "2026-02-15 04:35:20", + "2026-02-15 04:35:30", + "2026-02-15 04:35:40", + "2026-02-15 04:35:50", + "2026-02-15 04:36:00", + "2026-02-15 04:36:10", + "2026-02-15 04:36:20", + "2026-02-15 04:36:30", + "2026-02-15 04:36:40", + "2026-02-15 04:36:50", + "2026-02-15 04:37:00", + "2026-02-15 04:37:10", + "2026-02-15 04:37:20", + "2026-02-15 04:37:30", + "2026-02-15 04:37:40", + "2026-02-15 04:37:50", + "2026-02-15 04:38:00", + "2026-02-15 04:38:10", + "2026-02-15 04:38:20", + "2026-02-15 04:38:30", + "2026-02-15 04:38:40", + "2026-02-15 04:38:50", + "2026-02-15 04:39:00", + "2026-02-15 04:39:10", + "2026-02-15 04:39:20", + "2026-02-15 04:39:30", + "2026-02-15 04:39:40", + "2026-02-15 04:39:50", + "2026-02-15 04:40:00", + "2026-02-15 04:40:10", + "2026-02-15 04:40:20", + "2026-02-15 04:40:30", + "2026-02-15 04:40:40", + "2026-02-15 04:40:50", + "2026-02-15 04:41:00", + "2026-02-15 04:41:10", + "2026-02-15 04:41:20", + "2026-02-15 04:41:30", + "2026-02-15 04:41:40", + "2026-02-15 04:41:50", + "2026-02-15 04:42:00", + "2026-02-15 04:42:10", + "2026-02-15 04:42:20", + "2026-02-15 04:42:30", + "2026-02-15 04:42:40", + "2026-02-15 04:42:50", + "2026-02-15 04:43:00", + "2026-02-15 04:43:10", + "2026-02-15 04:43:20", + "2026-02-15 04:43:30", + "2026-02-15 04:43:40", + "2026-02-15 04:43:50", + "2026-02-15 04:44:00", + "2026-02-15 04:44:10", + "2026-02-15 04:44:20", + "2026-02-15 04:44:30", + "2026-02-15 04:44:40", + "2026-02-15 04:44:50", + "2026-02-15 04:45:00", + "2026-02-15 04:45:10", + "2026-02-15 04:45:20", + "2026-02-15 04:45:30", + "2026-02-15 04:45:40", + "2026-02-15 04:45:50", + "2026-02-15 04:46:00", + "2026-02-15 04:46:10", + "2026-02-15 04:46:20", + "2026-02-15 04:46:30", + "2026-02-15 04:46:40", + "2026-02-15 04:46:50", + "2026-02-15 04:47:00", + "2026-02-15 04:47:10", + "2026-02-15 04:47:20", + "2026-02-15 04:47:30", + "2026-02-15 04:47:40", + "2026-02-15 04:47:50", + "2026-02-15 04:48:00", + "2026-02-15 04:48:10", + "2026-02-15 04:48:20", + "2026-02-15 04:48:30", + "2026-02-15 04:48:40", + "2026-02-15 04:48:50", + "2026-02-15 04:49:00", + "2026-02-15 04:49:10", + "2026-02-15 04:49:20", + "2026-02-15 04:49:30", + "2026-02-15 04:49:40", + "2026-02-15 04:49:50", + "2026-02-15 04:50:00", + "2026-02-15 04:50:10", + "2026-02-15 04:50:20", + "2026-02-15 04:50:30", + "2026-02-15 04:50:40", + "2026-02-15 04:50:50", + "2026-02-15 04:51:00", + "2026-02-15 04:51:10", + "2026-02-15 04:51:20", + "2026-02-15 04:51:30", + "2026-02-15 04:51:40", + "2026-02-15 04:51:50", + "2026-02-15 04:52:00", + "2026-02-15 04:52:10", + "2026-02-15 04:52:20", + "2026-02-15 04:52:30", + "2026-02-15 04:52:40", + "2026-02-15 04:52:50", + "2026-02-15 04:53:00", + "2026-02-15 04:53:10", + "2026-02-15 04:53:20", + "2026-02-15 04:53:30", + "2026-02-15 04:53:40", + "2026-02-15 04:53:50", + "2026-02-15 04:54:00", + "2026-02-15 04:54:10", + "2026-02-15 04:54:20", + "2026-02-15 04:54:30", + "2026-02-15 04:54:40", + "2026-02-15 04:54:50", + "2026-02-15 04:55:00", + "2026-02-15 04:55:10", + "2026-02-15 04:55:20", + "2026-02-15 04:55:30", + "2026-02-15 04:55:40", + "2026-02-15 04:55:50", + "2026-02-15 04:56:00", + "2026-02-15 04:56:10", + "2026-02-15 04:56:20", + "2026-02-15 04:56:30", + "2026-02-15 04:56:40", + "2026-02-15 04:56:50", + "2026-02-15 04:57:00", + "2026-02-15 04:57:10", + "2026-02-15 04:57:20", + "2026-02-15 04:57:30", + "2026-02-15 04:57:40", + "2026-02-15 04:57:50", + "2026-02-15 04:58:00", + "2026-02-15 04:58:10", + "2026-02-15 04:58:20", + "2026-02-15 04:58:30", + "2026-02-15 04:58:40", + "2026-02-15 04:58:50", + "2026-02-15 04:59:00", + "2026-02-15 04:59:10", + "2026-02-15 04:59:20", + "2026-02-15 04:59:30", + "2026-02-15 04:59:40", + "2026-02-15 04:59:50", + "2026-02-15 05:00:00", + "2026-02-15 05:00:10", + "2026-02-15 05:00:20", + "2026-02-15 05:00:30", + "2026-02-15 05:00:40", + "2026-02-15 05:00:50", + "2026-02-15 05:01:00", + "2026-02-15 05:01:10", + "2026-02-15 05:01:20", + "2026-02-15 05:01:30", + "2026-02-15 05:01:40", + "2026-02-15 05:01:50", + "2026-02-15 05:02:00", + "2026-02-15 05:02:10", + "2026-02-15 05:02:20", + "2026-02-15 05:02:30", + "2026-02-15 05:02:40", + "2026-02-15 05:02:50", + "2026-02-15 05:03:00", + "2026-02-15 05:03:10", + "2026-02-15 05:03:20", + "2026-02-15 05:03:30", + "2026-02-15 05:03:40", + "2026-02-15 05:03:50", + "2026-02-15 05:04:00", + "2026-02-15 05:04:10", + "2026-02-15 05:04:20", + "2026-02-15 05:04:30", + "2026-02-15 05:04:40", + "2026-02-15 05:04:50", + "2026-02-15 05:05:00", + "2026-02-15 05:05:10", + "2026-02-15 05:05:20", + "2026-02-15 05:05:30", + "2026-02-15 05:05:40", + "2026-02-15 05:05:50", + "2026-02-15 05:06:00", + "2026-02-15 05:06:10", + "2026-02-15 05:06:20", + "2026-02-15 05:06:30", + "2026-02-15 05:06:40", + "2026-02-15 05:06:50", + "2026-02-15 05:07:00", + "2026-02-15 05:07:10", + "2026-02-15 05:07:20", + "2026-02-15 05:07:30", + "2026-02-15 05:07:40", + "2026-02-15 05:07:50", + "2026-02-15 05:08:00", + "2026-02-15 05:08:10", + "2026-02-15 05:08:20", + "2026-02-15 05:08:30", + "2026-02-15 05:08:40", + "2026-02-15 05:08:50", + "2026-02-15 05:09:00", + "2026-02-15 05:09:10", + "2026-02-15 05:09:20", + "2026-02-15 05:09:30", + "2026-02-15 05:09:40", + "2026-02-15 05:09:50", + "2026-02-15 05:10:00", + "2026-02-15 05:10:10", + "2026-02-15 05:10:20", + "2026-02-15 05:10:30", + "2026-02-15 05:10:40", + "2026-02-15 05:10:50", + "2026-02-15 05:11:00", + "2026-02-15 05:11:10", + "2026-02-15 05:11:20", + "2026-02-15 05:11:30", + "2026-02-15 05:11:40", + "2026-02-15 05:11:50", + "2026-02-15 05:12:00", + "2026-02-15 05:12:10", + "2026-02-15 05:12:20", + "2026-02-15 05:12:30", + "2026-02-15 05:12:40", + "2026-02-15 05:12:50", + "2026-02-15 05:13:00", + "2026-02-15 05:13:10", + "2026-02-15 05:13:20", + "2026-02-15 05:13:30", + "2026-02-15 05:13:40", + "2026-02-15 05:13:50", + "2026-02-15 05:14:00", + "2026-02-15 05:14:10", + "2026-02-15 05:14:20", + "2026-02-15 05:14:30", + "2026-02-15 05:14:40", + "2026-02-15 05:14:50", + "2026-02-15 05:15:00", + "2026-02-15 05:15:10", + "2026-02-15 05:15:20", + "2026-02-15 05:15:30", + "2026-02-15 05:15:40", + "2026-02-15 05:15:50", + "2026-02-15 05:16:00", + "2026-02-15 05:16:10", + "2026-02-15 05:16:20", + "2026-02-15 05:16:30", + "2026-02-15 05:16:40", + "2026-02-15 05:16:50", + "2026-02-15 05:17:00", + "2026-02-15 05:17:10", + "2026-02-15 05:17:20", + "2026-02-15 05:17:30", + "2026-02-15 05:17:40", + "2026-02-15 05:17:50", + "2026-02-15 05:18:00", + "2026-02-15 05:18:10", + "2026-02-15 05:18:20", + "2026-02-15 05:18:30", + "2026-02-15 05:18:40", + "2026-02-15 05:18:50", + "2026-02-15 05:19:00", + "2026-02-15 05:19:10", + "2026-02-15 05:19:20", + "2026-02-15 05:19:30", + "2026-02-15 05:19:40", + "2026-02-15 05:19:50", + "2026-02-15 05:20:00", + "2026-02-15 05:20:10", + "2026-02-15 05:20:20", + "2026-02-15 05:20:30", + "2026-02-15 05:20:40", + "2026-02-15 05:20:50", + "2026-02-15 05:21:00", + "2026-02-15 05:21:10", + "2026-02-15 05:21:20", + "2026-02-15 05:21:30", + "2026-02-15 05:21:40" + ], + "bi": [ + 314.182, + 281.441, + 225.199, + 156.098, + 0.0, + 0.8, + 0.0, + 0.0, + 0.0, + 12.6, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 3.6, + 1.4, + 1.1, + 5.0, + 6.5, + 0.9, + 0.9, + 1.0, + 0.0, + 2.2, + 0.0, + 0.0, + 16.7, + 53.299, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 10.0, + 0.0, + 0.0, + 2.199, + 1.7, + 2.101, + 2.3, + 438.634, + 0.0, + 93.499, + 1.0, + 0.3, + 0.0, + 0.1, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 216.805, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 537.567, + 0.3, + 0.0, + 0.2, + 0.0, + 0.0, + 11.897, + 2.1, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.6, + 92.8, + 1.7, + 0.3, + 101.404, + 204.198, + 0.0, + 0.9, + 0.3, + 0.0, + 0.0, + 0.0, + 0.0, + 0.6, + 0.0, + 0.0, + 57.192, + 107.6, + 6.7, + 0.7, + 202.876, + 190.612, + 1.5, + 1.4, + 1.3, + 0.7, + 0.0, + 173.105, + 0.0, + 3.3, + 0.5, + 0.8, + 0.0, + 0.0, + 0.0, + 0.6, + 0.0, + 0.0, + 125.192, + 0.6, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.4, + 0.2, + 0.9, + 0.0, + 122.904, + 0.1, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.1, + 58.198, + 510.708, + 0.0, + 0.0, + 0.0, + 0.6, + 593.459, + 82.507, + 0.0, + 0.0, + 29.796, + 0.1, + 121.005, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.2, + 0.5, + 0.4, + 0.0, + 0.0, + 11.999, + 8.102, + 0.0, + 0.0, + 1.2, + 0.0, + 0.0, + 0.4, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.1, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 2.5, + 0.4, + 0.2, + 0.2, + 0.2, + 0.2, + 0.2, + 76.799, + 43.299, + 129.895, + 35.597, + 45.599, + 61.042, + 591.309, + 449.077, + 748.822, + 376.611, + 1013.977, + 2531.182, + 1114.819, + 432.114, + 685.03, + 291.821, + 101.187, + 243.22, + 221.412, + 31.202, + 147.101, + 178.101, + 279.689, + 373.65, + 266.632, + 202.902, + 72.511, + 41.445, + 575.072, + 1099.016, + 593.246, + 588.361, + 591.057, + 352.853, + 363.697, + 104.693, + 239.926, + 344.703, + 126.697, + 236.877, + 64.924, + 143.37, + 771.711, + 355.07, + 165.931, + 39.7, + 81.701, + 451.115, + 205.302, + 359.468, + 537.92, + 269.443, + 720.503, + 338.888, + 534.709, + 36.401, + 210.702, + 326.716, + 630.591, + 544.513, + 1093.685, + 2030.018, + 2703.822, + 1850.553, + 1735.371, + 860.721, + 1150.817, + 642.043, + 1299.809, + 1941.834, + 1891.79, + 2819.127, + 679.777, + 499.503, + 1045.736, + 1292.77, + 1429.681, + 1009.835, + 2986.657, + 1299.771, + 1088.654, + 1755.576, + 1555.477, + 2608.098, + 2130.009, + 2290.086, + 1211.735, + 2190.327, + 1995.462, + 2272.947, + 2476.213, + 2209.12, + 2212.729, + 2029.749, + 2712.157, + 2130.152, + 3934.42, + 1872.944, + 2619.461, + 3302.736, + 3120.228, + 3551.598, + 2547.542, + 4246.125, + 3141.908, + 4129.963, + 3935.426, + 3814.9, + 2212.51, + 2641.736, + 3311.327, + 3040.041, + 1981.728, + 3026.68, + 3092.323, + 2024.272, + 2664.248, + 2287.127, + 2708.957, + 2512.686, + 3941.371, + 2787.839, + 2779.086, + 1993.933, + 2420.493, + 1406.921, + 1532.756, + 1197.563, + 869.07, + 1367.958, + 1255.669, + 947.322, + 947.592, + 1225.587, + 2230.582, + 1451.921, + 2055.192, + 2000.131, + 2685.934, + 3412.665, + 2683.137, + 2012.694, + 2985.723, + 3200.213, + 1825.68, + 2178.955, + 2208.453, + 1570.174, + 1623.524, + 458.371, + 2422.592, + 1143.582, + 664.605, + 883.504, + 1568.251, + 956.537, + 699.131, + 2630.19, + 1915.798, + 1441.891, + 964.632, + 1159.948, + 653.314, + 1481.8, + 1479.747, + 645.919, + 478.82, + 1119.819, + 337.716, + 375.505, + 540.792, + 1145.687, + 404.398, + 818.81, + 1682.253, + 2328.072, + 1684.685, + 2028.191, + 625.034, + 694.822, + 2359.328, + 1182.187, + 1822.246, + 2940.985, + 1249.327, + 1038.609, + 1466.346, + 336.301, + 255.703, + 615.181, + 372.225, + 645.67, + 1144.701, + 1159.948, + 666.532, + 934.661, + 764.259, + 727.827, + 1509.921, + 384.738, + 335.909, + 641.518, + 555.906, + 255.885, + 158.606, + 272.874, + 98.304, + 149.309, + 109.58, + 62.706, + 300.708, + 1902.433, + 1807.458, + 3582.848, + 5952.143, + 3093.432, + 2500.074, + 2049.69, + 2763.59, + 1230.553, + 1502.16, + 1934.505, + 1161.196, + 645.799, + 538.541, + 987.785, + 1984.646, + 858.001, + 406.003, + 197.496, + 119.801, + 325.635, + 1434.035, + 931.618, + 771.41, + 1527.883, + 1212.479, + 955.672, + 429.548, + 437.213, + 368.129, + 279.412, + 158.499, + 240.302, + 280.556, + 278.351, + 111.795, + 158.404, + 90.385, + 185.923, + 522.327, + 179.275, + 204.912, + 693.047, + 1907.578, + 1945.097, + 1439.302, + 1385.338, + 2326.465, + 1259.902, + 1720.949, + 1700.479, + 632.173, + 148.246, + 142.604, + 137.275, + 510.346, + 668.297, + 1040.949, + 885.612, + 1421.206, + 1306.352, + 458.228, + 54.608, + 113.005, + 44.799, + 160.796, + 83.903, + 59.889, + 90.313, + 62.503, + 43.701, + 39.598, + 51.703, + 39.5, + 76.988, + 35.903, + 43.2, + 36.401, + 42.201, + 21.7, + 76.994, + 19.601, + 256.398, + 48.5, + 45.702, + 41.399, + 20.999, + 25.401, + 27.801, + 71.997, + 45.101, + 13.8, + 33.298, + 34.299, + 41.004, + 24.4, + 24.499, + 21.401, + 24.5, + 48.491, + 36.006, + 33.699, + 34.102, + 22.2, + 25.499, + 33.697, + 154.917, + 53.701, + 81.601, + 222.896, + 67.202, + 27.0, + 19.897, + 47.406, + 14.401, + 32.699, + 57.5, + 54.501, + 41.1, + 47.39, + 114.215, + 828.508, + 1252.601, + 621.107, + 686.311, + 1415.908, + 648.625, + 165.93, + 284.506, + 290.198, + 115.101, + 233.186, + 231.312, + 113.38, + 228.417, + 86.408, + 45.102, + 126.091, + 32.202, + 32.694, + 104.814, + 306.909, + 201.105, + 121.694, + 84.804, + 107.3, + 137.273, + 101.913, + 40.102, + 51.8, + 390.983, + 90.306, + 232.2, + 83.387, + 72.506, + 48.702, + 43.001, + 40.3, + 64.498, + 74.803, + 65.885, + 48.808, + 22.401, + 18.2, + 55.597, + 44.999, + 127.59, + 46.304, + 75.106, + 23.1, + 70.6, + 31.7, + 131.268, + 816.707, + 313.324, + 106.803, + 805.288, + 378.751, + 764.283, + 93.886, + 28.204, + 20.201, + 17.3, + 33.401, + 32.699, + 45.5, + 59.891, + 30.103, + 31.601, + 20.4, + 34.0, + 20.6, + 36.2, + 22.398, + 29.798, + 16.002, + 53.5, + 46.2, + 17.7, + 25.197, + 18.701, + 28.701, + 233.699, + 59.699, + 76.502, + 171.899, + 330.06, + 271.007, + 18.501, + 71.603, + 69.298, + 956.212, + 522.201, + 572.431, + 261.12, + 762.569, + 269.127, + 399.127, + 99.512, + 162.881, + 26.804, + 64.901, + 62.899, + 166.091, + 41.101, + 80.295, + 64.602, + 51.6, + 47.803, + 262.697, + 32.999, + 26.802, + 38.396, + 135.802, + 1363.831, + 1188.323, + 1816.697, + 1498.484, + 791.74, + 1439.291, + 280.039, + 204.011, + 227.097, + 1297.557, + 1170.661, + 1815.354, + 601.767, + 171.913, + 275.395, + 137.291, + 43.403, + 98.991, + 84.099, + 50.405, + 50.896, + 116.609, + 36.4, + 52.2, + 56.301, + 77.287, + 461.619, + 28.102, + 40.901, + 27.098, + 80.907, + 25.196, + 46.405, + 50.702, + 132.702, + 154.896, + 29.099, + 61.202, + 42.793, + 35.003, + 34.001, + 56.599, + 465.633, + 666.174, + 374.894, + 193.176, + 192.808, + 93.609, + 121.203, + 91.199, + 256.159, + 283.315, + 97.306, + 106.201, + 75.704, + 126.895, + 63.601, + 34.401, + 79.088, + 67.705, + 45.502, + 133.299, + 160.699, + 93.6, + 48.593, + 105.31, + 89.001, + 163.708, + 65.999, + 118.304, + 47.7, + 62.595, + 61.101, + 43.501, + 72.5, + 119.805, + 39.6, + 106.795, + 77.691, + 35.804, + 44.701, + 44.102, + 109.992, + 22.401 + ], + "bo": [ + 47.334, + 521.748, + 1579.091, + 335.595, + 71.202, + 1520.814, + 267.499, + 94.701, + 66.4, + 129.595, + 330.307, + 5.9, + 177.603, + 2.3, + 71.499, + 111.101, + 9.6, + 86.7, + 90.2, + 304.589, + 83.302, + 1018.375, + 186.598, + 777.706, + 310.017, + 229.287, + 460.497, + 139.707, + 335.184, + 75.303, + 135.299, + 799.911, + 226.895, + 165.203, + 6.9, + 1804.013, + 5953.44, + 5133.944, + 2244.609, + 468.608, + 5118.564, + 6039.642, + 2918.704, + 5125.137, + 3857.815, + 6627.68, + 6024.049, + 6104.578, + 6399.628, + 7948.899, + 7142.418, + 4601.245, + 646.071, + 3602.707, + 4855.175, + 95.801, + 23.6, + 2587.271, + 1602.971, + 1.6, + 512.209, + 796.176, + 340.601, + 314.972, + 3318.957, + 1102.684, + 1324.234, + 1959.363, + 979.222, + 665.592, + 0.2, + 0.3, + 60.202, + 0.4, + 3.8, + 72.197, + 283.379, + 41.404, + 269.576, + 27.603, + 2009.309, + 183.6, + 541.277, + 96.308, + 72.905, + 64.0, + 96.201, + 168.697, + 607.818, + 137.498, + 907.812, + 87.613, + 660.729, + 1082.13, + 518.007, + 265.126, + 334.893, + 1272.277, + 809.004, + 3105.208, + 1380.061, + 394.898, + 329.298, + 559.859, + 529.721, + 1074.689, + 559.614, + 255.305, + 211.302, + 926.646, + 2035.021, + 356.078, + 463.69, + 323.337, + 203.802, + 239.31, + 517.727, + 592.497, + 181.706, + 100.707, + 738.711, + 1041.364, + 317.703, + 541.561, + 970.028, + 937.552, + 217.408, + 5.5, + 943.476, + 338.586, + 7.401, + 894.682, + 1012.572, + 705.232, + 906.476, + 212.541, + 21.804, + 1.2, + 17.599, + 96.003, + 161.0, + 337.606, + 57.501, + 5.399, + 535.824, + 2119.799, + 379.897, + 224.476, + 865.93, + 87.209, + 348.369, + 81.703, + 360.995, + 369.721, + 156.507, + 568.883, + 376.683, + 903.583, + 267.51, + 316.602, + 103.908, + 365.2, + 2354.728, + 85.302, + 131.29, + 263.097, + 152.116, + 116.601, + 155.995, + 37.001, + 822.197, + 231.684, + 214.909, + 12.1, + 114.592, + 0.0, + 3.7, + 298.388, + 22.597, + 1683.789, + 2003.289, + 3342.105, + 62.198, + 114.906, + 0.1, + 1.6, + 8.7, + 17.601, + 87.799, + 12.5, + 73.699, + 136.988, + 27.402, + 76.501, + 73.6, + 63.799, + 206.3, + 341.166, + 246.485, + 333.807, + 170.004, + 453.479, + 640.427, + 647.53, + 405.712, + 168.685, + 150.106, + 931.381, + 2660.58, + 116.404, + 971.473, + 168.784, + 1047.342, + 384.012, + 230.604, + 1050.197, + 357.191, + 249.097, + 645.341, + 1143.697, + 429.613, + 198.706, + 287.187, + 110.502, + 237.982, + 151.409, + 1.2, + 0.4, + 1365.718, + 22.4, + 21.9, + 50.398, + 3.1, + 989.63, + 1046.708, + 6069.861, + 6938.535, + 6997.188, + 2712.007, + 669.975, + 1846.42, + 736.435, + 93.701, + 39.297, + 173.501, + 964.779, + 2055.841, + 2875.275, + 1919.149, + 840.069, + 1741.555, + 3795.825, + 4216.495, + 5545.475, + 5746.406, + 4958.747, + 3992.021, + 3135.819, + 5369.486, + 5427.193, + 5852.781, + 6936.831, + 6417.361, + 6463.15, + 6178.008, + 6172.331, + 6346.979, + 6175.329, + 6195.118, + 6228.466, + 6078.081, + 6328.449, + 6343.96, + 6194.878, + 6260.752, + 5915.716, + 5631.72, + 6139.041, + 6060.625, + 5912.07, + 6329.342, + 6239.656, + 6437.755, + 6159.875, + 6070.359, + 6357.326, + 6327.781, + 6474.953, + 6424.108, + 5762.382, + 6135.282, + 6270.473, + 6318.869, + 6209.642, + 6053.797, + 6203.551, + 6072.499, + 6094.99, + 6449.444, + 5970.11, + 6171.978, + 6201.903, + 6391.637, + 6369.073, + 6735.529, + 6154.759, + 6108.035, + 6238.313, + 5261.507, + 4686.212, + 4978.165, + 5169.939, + 5925.161, + 5770.276, + 6185.701, + 5658.467, + 5229.014, + 3171.121, + 2413.181, + 1499.87, + 1125.608, + 1292.721, + 2693.138, + 2975.861, + 4259.57, + 6449.681, + 5175.579, + 5809.82, + 7029.306, + 6297.303, + 6432.341, + 5631.817, + 6235.533, + 5875.469, + 5305.122, + 5518.143, + 4755.907, + 5598.429, + 5264.885, + 5328.119, + 5210.912, + 4687.062, + 5335.031, + 2199.255, + 976.075, + 3123.453, + 1365.715, + 2188.72, + 2467.621, + 2539.441, + 2540.515, + 2865.916, + 3181.703, + 1488.021, + 3539.579, + 2430.68, + 2190.43, + 2340.09, + 1621.275, + 1675.023, + 3360.611, + 5232.54, + 5588.593, + 4494.713, + 5188.128, + 5305.653, + 5437.351, + 3623.51, + 4313.474, + 3300.277, + 3409.414, + 2814.876, + 1221.592, + 1088.169, + 2434.227, + 4101.233, + 2526.992, + 2960.326, + 4118.493, + 3345.272, + 2102.178, + 5763.919, + 5947.506, + 2497.49, + 4071.564, + 3105.508, + 4442.094, + 4642.317, + 5082.896, + 4472.693, + 3159.864, + 2486.5, + 3308.831, + 2757.016, + 2384.909, + 1470.331, + 1299.902, + 2300.503, + 1989.043, + 2228.052, + 3231.85, + 2989.898, + 3867.855, + 4063.202, + 5425.487, + 4496.729, + 2723.282, + 4211.165, + 3621.763, + 2275.75, + 2236.899, + 764.825, + 1084.632, + 789.968, + 623.966, + 1000.746, + 740.509, + 597.791, + 1185.187, + 1057.494, + 848.606, + 1266.94, + 863.864, + 864.544, + 944.849, + 819.213, + 656.921, + 755.141, + 838.191, + 856.122, + 1062.495, + 1346.529, + 1173.197, + 996.999, + 1028.303, + 877.709, + 737.078, + 800.953, + 770.945, + 1448.301, + 1223.656, + 1258.66, + 1322.344, + 1185.436, + 1280.247, + 1247.752, + 1131.412, + 823.821, + 784.022, + 763.508, + 785.453, + 966.334, + 668.036, + 667.524, + 764.946, + 637.981, + 567.652, + 758.219, + 777.591, + 1301.986, + 2294.895, + 2193.405, + 2855.022, + 3085.545, + 1651.072, + 2613.613, + 2732.539, + 1789.672, + 1635.189, + 858.897, + 995.799, + 700.623, + 757.165, + 718.517, + 495.401, + 611.605, + 617.087, + 457.603, + 697.161, + 767.819, + 757.896, + 865.311, + 710.439, + 769.787, + 680.68, + 797.604, + 520.315, + 791.063, + 721.631, + 720.097, + 684.006, + 679.894, + 876.56, + 768.165, + 591.413, + 828.262, + 659.881, + 578.33, + 762.795, + 532.332, + 870.559, + 857.39, + 780.299, + 849.801, + 982.256, + 792.254, + 822.267, + 1236.135, + 1539.671, + 995.713, + 1405.889, + 715.518, + 684.075, + 643.859, + 611.097, + 634.53, + 815.211, + 731.503, + 649.176, + 678.393, + 522.277, + 512.923, + 702.189, + 471.288, + 574.32, + 560.997, + 547.579, + 558.127, + 504.206, + 555.169, + 499.131, + 535.697, + 583.51, + 489.74, + 630.205, + 511.118, + 542.61, + 563.493, + 491.863, + 573.223, + 527.396, + 524.795, + 561.825, + 512.888, + 563.377, + 538.03, + 530.417, + 673.569, + 520.809, + 565.517, + 584.656, + 538.382, + 610.259, + 590.999, + 589.588, + 525.723, + 559.991, + 583.787, + 478.879, + 644.182, + 533.731, + 561.195, + 539.974, + 530.453, + 643.271, + 626.508, + 631.71, + 819.685, + 762.521, + 536.099, + 480.325, + 675.88, + 578.227, + 490.782, + 719.196, + 567.614, + 544.706, + 696.254, + 492.666, + 566.705, + 914.401, + 658.308, + 482.108, + 858.244, + 825.204, + 460.683, + 758.815, + 515.496, + 620.503, + 633.962, + 601.932, + 676.783, + 540.64, + 514.248, + 577.226, + 549.663, + 477.727, + 637.49, + 519.568, + 580.917, + 615.914, + 487.076, + 575.429, + 607.403, + 491.003, + 688.185, + 551.026, + 683.194, + 641.171, + 701.243, + 629.9, + 604.705, + 707.957, + 581.228, + 484.306, + 637.899, + 530.887, + 463.221, + 569.373, + 470.776, + 524.832, + 592.901, + 488.174, + 570.085, + 565.654, + 541.25, + 721.359, + 495.39, + 603.803, + 639.003, + 534.469, + 505.966, + 683.253, + 505.813, + 756.189, + 589.623, + 706.877, + 566.617, + 504.168, + 616.335, + 540.392, + 592.611, + 570.99, + 597.406, + 527.52, + 644.056, + 572.512, + 530.107, + 640.902, + 515.899, + 542.507, + 570.161, + 547.562, + 557.77, + 590.704, + 568.995, + 575.999, + 488.849, + 582.337, + 563.329, + 487.797, + 672.893, + 531.715, + 531.997, + 600.828, + 627.017, + 527.33, + 583.623, + 618.481, + 754.41, + 677.202, + 808.002, + 642.95, + 580.676, + 660.966, + 719.568, + 574.27, + 892.694, + 602.99, + 537.707, + 955.878, + 576.07, + 682.025, + 800.348, + 755.723, + 496.203, + 728.841, + 626.593, + 613.38, + 743.149, + 616.431, + 757.311, + 825.158, + 866.189, + 739.099, + 742.842, + 796.34, + 1066.345, + 607.186, + 584.33, + 690.792, + 902.07, + 686.636, + 758.797, + 656.773, + 670.849, + 469.391, + 575.161, + 549.341, + 577.448, + 486.597, + 611.76, + 493.164, + 515.238, + 529.906, + 557.104, + 539.707, + 481.319, + 543.922, + 529.342, + 569.114, + 522.169, + 519.343, + 536.318, + 560.355, + 498.524, + 579.808, + 520.887, + 520.19, + 491.216, + 561.608, + 538.649, + 526.02, + 565.289, + 417.23, + 700.973, + 598.691, + 509.637, + 480.42, + 658.96, + 540.214, + 433.597, + 649.695, + 571.931, + 522.834, + 562.005, + 500.829, + 553.877, + 515.708, + 468.719, + 531.518, + 559.139, + 453.824, + 591.096, + 526.196, + 547.402, + 511.829, + 505.346, + 528.808, + 482.323, + 576.691, + 546.617, + 575.598, + 501.357, + 519.113, + 479.506, + 597.699, + 464.219, + 598.903, + 473.679, + 523.44, + 590.763, + 462.708, + 549.224, + 577.256, + 443.929 + ], + "await": [ + 0.615, + 1.46, + 2.275, + 1.178, + 1.923, + 1.241, + 1.473, + 1.927, + 1.932, + 1.766, + 1.919, + 1.051, + 1.447, + 1.174, + 1.923, + 1.839, + 1.823, + 1.93, + 1.845, + 1.437, + 1.519, + 2.089, + 1.094, + 1.912, + 1.351, + 1.783, + 1.676, + 1.019, + 1.627, + 1.495, + 0.997, + 1.068, + 1.114, + 1.234, + 1.87, + 2.007, + 1.994, + 2.017, + 2.015, + 2.015, + 2.022, + 2.024, + 2.021, + 2.024, + 2.016, + 3.003, + 2.028, + 2.891, + 3.545, + 3.522, + 4.032, + 3.15, + 1.435, + 1.607, + 1.586, + 1.444, + 0.836, + 1.448, + 1.344, + 1.25, + 1.359, + 1.023, + 1.016, + 1.017, + 1.124, + 1.363, + 1.482, + 1.61, + 1.512, + 1.51, + 1.5, + 1.333, + 1.917, + 1.0, + 1.252, + 0.927, + 0.905, + 1.791, + 1.442, + 1.051, + 2.469, + 1.291, + 0.888, + 0.892, + 0.963, + 0.919, + 0.886, + 0.913, + 0.889, + 0.895, + 1.81, + 0.904, + 0.891, + 1.01, + 0.909, + 1.137, + 1.416, + 0.934, + 1.585, + 1.552, + 1.555, + 1.488, + 1.609, + 1.802, + 1.513, + 1.623, + 1.524, + 0.955, + 1.041, + 1.647, + 1.872, + 0.968, + 0.902, + 0.905, + 1.385, + 0.906, + 1.315, + 1.383, + 1.617, + 0.939, + 1.307, + 1.58, + 1.482, + 1.11, + 1.425, + 1.548, + 1.084, + 0.717, + 1.434, + 1.372, + 1.38, + 1.306, + 1.209, + 0.889, + 0.895, + 1.223, + 1.872, + 1.333, + 0.746, + 0.903, + 0.909, + 1.738, + 1.817, + 1.111, + 1.712, + 1.977, + 1.395, + 0.987, + 1.96, + 1.728, + 1.573, + 1.179, + 1.426, + 1.636, + 1.727, + 1.908, + 1.221, + 1.637, + 1.074, + 1.057, + 1.291, + 1.375, + 2.18, + 1.319, + 1.232, + 1.17, + 1.256, + 1.334, + 1.049, + 0.751, + 1.602, + 1.478, + 1.094, + 0.976, + 0.749, + 0.588, + 1.135, + 1.388, + 1.013, + 1.939, + 1.778, + 2.988, + 1.087, + 1.023, + 1.0, + 1.0, + 1.149, + 1.83, + 1.904, + 1.088, + 1.889, + 1.914, + 1.788, + 1.871, + 1.908, + 1.561, + 1.385, + 1.545, + 1.147, + 1.797, + 1.355, + 1.1, + 1.685, + 1.138, + 1.111, + 1.606, + 1.779, + 1.489, + 2.587, + 1.76, + 1.135, + 1.289, + 1.819, + 1.428, + 1.585, + 1.88, + 1.438, + 1.315, + 1.78, + 1.315, + 1.337, + 1.591, + 1.237, + 1.744, + 1.57, + 1.133, + 1.083, + 1.0, + 1.798, + 1.915, + 1.886, + 1.056, + 1.194, + 1.174, + 1.091, + 1.438, + 1.68, + 1.71, + 1.821, + 1.886, + 1.896, + 1.61, + 1.396, + 1.134, + 1.206, + 1.776, + 1.724, + 1.501, + 1.517, + 1.641, + 1.701, + 1.769, + 1.697, + 1.508, + 1.533, + 1.578, + 1.573, + 1.5, + 1.767, + 1.634, + 1.569, + 1.906, + 2.056, + 1.976, + 2.24, + 2.13, + 1.994, + 2.381, + 2.995, + 1.688, + 1.974, + 2.803, + 3.529, + 2.094, + 2.059, + 3.139, + 2.579, + 2.345, + 3.27, + 2.327, + 3.031, + 3.09, + 2.006, + 2.114, + 2.149, + 2.015, + 1.948, + 1.927, + 1.975, + 2.323, + 2.131, + 1.993, + 1.911, + 1.998, + 2.218, + 2.04, + 2.114, + 2.168, + 1.918, + 2.24, + 2.129, + 2.208, + 1.972, + 2.016, + 2.059, + 2.245, + 1.643, + 1.518, + 3.281, + 4.128, + 5.019, + 4.656, + 3.803, + 3.053, + 2.211, + 2.621, + 2.824, + 1.306, + 1.119, + 1.258, + 1.124, + 1.031, + 1.375, + 1.37, + 1.458, + 1.44, + 1.438, + 1.495, + 1.458, + 1.451, + 1.688, + 1.381, + 1.436, + 2.634, + 4.003, + 3.666, + 5.236, + 2.703, + 3.752, + 3.941, + 5.057, + 4.643, + 2.904, + 1.038, + 0.932, + 1.217, + 1.051, + 1.121, + 1.033, + 1.052, + 0.926, + 1.053, + 0.981, + 3.707, + 1.176, + 1.153, + 1.091, + 1.138, + 1.02, + 1.093, + 1.075, + 1.318, + 2.62, + 3.071, + 1.837, + 1.673, + 2.703, + 5.66, + 3.909, + 1.145, + 1.253, + 1.189, + 1.012, + 0.972, + 1.325, + 1.393, + 1.307, + 1.339, + 1.35, + 1.43, + 1.272, + 1.289, + 2.494, + 1.12, + 1.165, + 1.17, + 3.51, + 4.468, + 3.558, + 5.057, + 1.169, + 1.225, + 1.174, + 1.167, + 1.098, + 1.026, + 1.11, + 1.044, + 1.286, + 1.258, + 1.296, + 1.338, + 1.38, + 1.323, + 1.404, + 1.975, + 1.251, + 1.448, + 1.305, + 1.244, + 1.222, + 0.853, + 0.964, + 0.872, + 0.826, + 0.885, + 0.913, + 0.869, + 0.953, + 0.949, + 0.89, + 0.946, + 0.87, + 0.841, + 0.86, + 0.852, + 0.879, + 0.839, + 0.875, + 0.893, + 1.105, + 1.046, + 0.994, + 1.083, + 1.042, + 0.942, + 0.929, + 0.888, + 0.866, + 1.013, + 1.077, + 1.188, + 1.076, + 1.077, + 1.149, + 0.963, + 1.104, + 1.024, + 0.875, + 0.905, + 0.993, + 1.026, + 0.881, + 0.902, + 0.899, + 0.92, + 0.894, + 0.856, + 0.873, + 0.948, + 1.031, + 1.826, + 1.095, + 1.149, + 1.027, + 1.142, + 1.189, + 1.085, + 1.014, + 0.919, + 1.035, + 0.876, + 0.851, + 0.879, + 0.846, + 0.883, + 0.86, + 0.879, + 0.881, + 0.846, + 0.905, + 0.873, + 0.842, + 0.84, + 0.845, + 0.845, + 0.837, + 0.87, + 0.863, + 0.872, + 0.898, + 0.863, + 0.879, + 0.881, + 0.893, + 0.892, + 0.883, + 0.82, + 0.869, + 0.88, + 0.865, + 0.916, + 0.871, + 0.906, + 0.975, + 0.96, + 0.905, + 1.045, + 1.179, + 1.034, + 1.24, + 0.904, + 0.883, + 0.877, + 0.903, + 0.859, + 0.956, + 0.924, + 0.96, + 0.886, + 0.903, + 0.872, + 0.895, + 0.861, + 0.887, + 0.885, + 0.897, + 0.881, + 0.897, + 0.906, + 0.881, + 0.888, + 0.892, + 0.886, + 0.895, + 0.911, + 0.881, + 0.901, + 0.889, + 0.892, + 0.856, + 0.892, + 0.884, + 0.891, + 0.9, + 0.897, + 0.899, + 0.896, + 0.898, + 0.904, + 0.892, + 0.888, + 0.898, + 0.888, + 0.894, + 0.897, + 0.889, + 0.886, + 0.895, + 0.886, + 0.906, + 0.895, + 0.896, + 0.898, + 0.878, + 0.886, + 0.898, + 1.04, + 0.924, + 0.898, + 0.895, + 0.893, + 0.9, + 0.886, + 0.896, + 0.896, + 0.893, + 0.892, + 0.886, + 0.85, + 0.956, + 0.884, + 0.859, + 0.9, + 1.001, + 0.874, + 0.911, + 0.901, + 1.06, + 0.888, + 0.902, + 1.051, + 0.871, + 0.877, + 0.904, + 0.901, + 0.891, + 0.905, + 0.878, + 0.848, + 0.873, + 0.875, + 0.877, + 0.892, + 0.879, + 0.878, + 0.901, + 0.901, + 0.955, + 0.897, + 0.869, + 0.884, + 0.894, + 0.897, + 0.885, + 0.895, + 0.884, + 0.879, + 0.89, + 0.89, + 0.895, + 0.905, + 0.907, + 0.887, + 0.89, + 0.882, + 0.882, + 0.906, + 0.882, + 0.898, + 0.935, + 0.852, + 0.915, + 0.875, + 0.956, + 0.919, + 0.939, + 0.876, + 0.903, + 0.899, + 0.895, + 0.902, + 0.893, + 0.896, + 0.892, + 0.894, + 0.897, + 0.905, + 0.892, + 0.906, + 0.892, + 0.896, + 0.902, + 0.886, + 0.9, + 0.907, + 0.889, + 0.889, + 0.905, + 0.894, + 0.844, + 0.902, + 0.882, + 0.895, + 0.85, + 0.855, + 0.901, + 0.9, + 0.901, + 0.878, + 0.924, + 0.891, + 0.863, + 0.855, + 0.88, + 0.88, + 0.891, + 0.906, + 0.905, + 0.922, + 0.906, + 0.893, + 0.91, + 0.916, + 0.898, + 0.89, + 0.925, + 0.869, + 0.899, + 0.919, + 0.917, + 0.936, + 0.822, + 0.963, + 0.878, + 1.047, + 0.951, + 0.977, + 0.872, + 0.881, + 0.96, + 0.927, + 0.925, + 0.86, + 0.897, + 0.952, + 0.852, + 0.881, + 0.955, + 0.876, + 0.881, + 0.891, + 0.884, + 0.874, + 0.915, + 0.907, + 0.891, + 0.874, + 0.969, + 0.895, + 0.898, + 0.888, + 0.927, + 0.892, + 0.9, + 0.896, + 0.878, + 0.877, + 0.896, + 0.9, + 0.891, + 0.894, + 0.898, + 0.906, + 0.865, + 0.969, + 0.913, + 0.858, + 0.943, + 0.964, + 0.884, + 0.884, + 0.905, + 0.955, + 0.912, + 0.882, + 0.888, + 0.894, + 0.883, + 0.926, + 0.899, + 0.909, + 0.904, + 0.893, + 0.881, + 0.898, + 0.887, + 0.885, + 0.881, + 0.928, + 0.895, + 0.912, + 0.905, + 0.899, + 0.895, + 0.909, + 0.892, + 0.933, + 0.899, + 0.938, + 0.894, + 0.903, + 0.875, + 0.888, + 0.941, + 0.896 + ] +} \ No newline at end of file diff --git a/.claude/scripts/io-performance-analyzer/parse_pcp.py b/.claude/scripts/io-performance-analyzer/parse_pcp.py new file mode 100644 index 0000000000..e2e75b4180 --- /dev/null +++ b/.claude/scripts/io-performance-analyzer/parse_pcp.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Parse pcp2json output with disk.dev.read, disk.dev.write, and disk.dev.await. + +pcp2json produces a nested structure: + @pcp.@hosts[].@metrics[] where each metric entry has @timestamp and + nested disk.dev.{read,write,await}.@instances[].{name, value}. + +Aggregates per-device instances: sum for read/write, max for await. +Outputs a clean JSON with arrays: timestamps, bi, bo, await. +""" + +import argparse +import json +import sys +from datetime import datetime +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +FMT = "%Y-%m-%d %H:%M:%S" + + +def get_instances(sample, *path): + """Walk the nested dict to reach @instances list for a metric.""" + node = sample + for key in path: + if not isinstance(node, dict) or key not in node: + return [] + node = node[key] + return node.get("@instances", []) + + +def aggregate_instances(instances, func): + """Aggregate instance values, converting strings to float.""" + vals = [] + for inst in instances: + try: + vals.append(float(inst["value"])) + except (KeyError, ValueError, TypeError): + continue + return func(vals) if vals else 0 + + +def convert_timestamp(ts_str, target_tz): + """Parse a pcp2json timestamp and convert to the target timezone.""" + utc = ZoneInfo("UTC") + for fmt in (FMT, "%Y-%m-%d %H:%M:%S%z", "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%dT%H:%M:%S%z"): + try: + ts = datetime.strptime(ts_str, fmt) + break + except ValueError: + continue + else: + return None + if ts.tzinfo is None: + ts = ts.replace(tzinfo=utc) + return ts.astimezone(target_tz).strftime(FMT) + + +def main(): + parser = argparse.ArgumentParser( + description="Parse pcp2json output into plot-ready JSON") + parser.add_argument("input_json", help="Raw pcp2json output file") + parser.add_argument("output_json", help="Output JSON file") + parser.add_argument("--timezone", default="UTC", + help="Target timezone for timestamps (default: UTC)") + args = parser.parse_args() + + try: + target_tz = ZoneInfo(args.timezone) + except (KeyError, ZoneInfoNotFoundError): + print(f"ERROR: Unknown timezone '{args.timezone}'", file=sys.stderr) + sys.exit(1) + + with open(args.input_json) as f: + raw = f.read() + + # Strip pcp2json comment lines (lines starting with { "//" ... }) + lines = [] + for line in raw.splitlines(): + stripped = line.strip() + if stripped.startswith('{ "//":') and stripped.endswith("}"): + continue + lines.append(line) + raw = "\n".join(lines) + + try: + data = json.loads(raw) + except json.JSONDecodeError as e: + print(f"ERROR: Failed to parse pcp2json output: {e}", file=sys.stderr) + sys.exit(1) + + # Navigate to the metrics array + hosts = data.get("@pcp", {}).get("@hosts", []) + if not hosts: + print("ERROR: No hosts found in pcp2json output", file=sys.stderr) + sys.exit(1) + + samples = hosts[0].get("@metrics", []) + if not samples: + print("ERROR: No metric samples found in pcp2json output", + file=sys.stderr) + sys.exit(1) + + result = {"timestamps": [], "bi": [], "bo": [], "await": []} + + for sample in samples: + ts_str = sample.get("@timestamp", "") + ts = convert_timestamp(ts_str, target_tz) + if ts is None: + continue + + read_insts = get_instances(sample, "disk", "dev", "read") + write_insts = get_instances(sample, "disk", "dev", "write") + await_insts = get_instances(sample, "disk", "dev", "await") + + # Skip samples with no metric data (e.g. first interval) + if not read_insts and not write_insts and not await_insts: + continue + + result["timestamps"].append(ts) + result["bi"].append(aggregate_instances(read_insts, sum)) + result["bo"].append(aggregate_instances(write_insts, sum)) + result["await"].append(aggregate_instances(await_insts, max)) + + if not result["timestamps"]: + print("ERROR: No valid data points found", file=sys.stderr) + sys.exit(1) + + with open(args.output_json, "w") as f: + json.dump(result, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/.claude/scripts/io-performance-analyzer/plot_io.py b/.claude/scripts/io-performance-analyzer/plot_io.py new file mode 100755 index 0000000000..a6a6277d20 --- /dev/null +++ b/.claude/scripts/io-performance-analyzer/plot_io.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Generate Disk IO Performance graph from pcp2json-derived data. + +Reads a JSON file with arrays: timestamps, bi, bo, await. +Produces a line chart of Disk Read OPS, Disk Write OPS, and Disk Await. +""" + +import argparse +import json +import sys +from datetime import datetime + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 +import matplotlib.dates as mdates # noqa: E402 + +FMT = "%Y-%m-%d %H:%M:%S" + + +def main(): + parser = argparse.ArgumentParser(description="Plot disk IO from JSON data") + parser.add_argument("json_file", help="Path to JSON file with timestamps/bi/bo/await") + parser.add_argument("-o", "--output", default="/data/disk_io_performance.png", + help="Output PNG file path") + parser.add_argument("--timezone", default="UTC", + help="Timezone label for the chart (default: UTC)") + args = parser.parse_args() + + with open(args.json_file) as f: + data = json.load(f) + + timestamps = [] + bi_values = [] + bo_values = [] + await_values = [] + + for i, ts_str in enumerate(data["timestamps"]): + try: + ts = datetime.strptime(ts_str, FMT) + except ValueError: + continue + timestamps.append(ts) + bi_values.append(float(data["bi"][i])) + bo_values.append(float(data["bo"][i])) + await_values.append(float(data["await"][i])) + + if not timestamps: + print("ERROR: No data points found in JSON", file=sys.stderr) + sys.exit(1) + + print(f"Loaded {len(timestamps)} data points") + + # Find peak values + bi_peak_idx = max(range(len(bi_values)), key=lambda i: bi_values[i]) + bo_peak_idx = max(range(len(bo_values)), key=lambda i: bo_values[i]) + q_peak_idx = max(range(len(await_values)), key=lambda i: await_values[i]) + bi_peak_ts = timestamps[bi_peak_idx].strftime("%H:%M:%S") + bo_peak_ts = timestamps[bo_peak_idx].strftime("%H:%M:%S") + q_peak_ts = timestamps[q_peak_idx].strftime("%H:%M:%S") + bi_peak_val = bi_values[bi_peak_idx] + bo_peak_val = bo_values[bo_peak_idx] + q_peak_val = await_values[q_peak_idx] + + print(f"Peak bi (read): {bi_peak_val:,.0f} blk/s at {bi_peak_ts}") + print(f"Peak bo (write): {bo_peak_val:,.0f} blk/s at {bo_peak_ts}") + print(f"Peak await: {q_peak_val:,.1f} ms at {q_peak_ts}") + + fig, (ax, ax_tbl) = plt.subplots( + 2, 1, figsize=(16, 7), + gridspec_kw={"height_ratios": [8, 1]}, + ) + + ax.plot(timestamps, bi_values, label="Disk Read OPS (bi)", color="tab:blue", + linewidth=0.8, alpha=0.9) + ax.plot(timestamps, bo_values, label="Disk Write OPS (bo)", color="tab:red", + linewidth=0.8, alpha=0.9) + + ax.set_xlabel(f"Time ({args.timezone})") + ax.set_ylabel("Block I/O (blocks/s)") + ax.set_title("Disk I/O Performance (15-second intervals)") + ax.grid(True, alpha=0.3) + + # Disk await on secondary Y axis + ax2 = ax.twinx() + ax2.plot(timestamps, await_values, label="Disk Await (ms)", color="tab:green", + linewidth=0.8, alpha=0.7, linestyle="--") + ax2.set_ylabel("Disk Await (ms)") + + # Combined legend + lines1, labels1 = ax.get_legend_handles_labels() + lines2, labels2 = ax2.get_legend_handles_labels() + ax.legend(lines1 + lines2, labels1 + labels2, loc="upper right") + + ax.xaxis.set_major_formatter(mdates.DateFormatter("%H:%M")) + ax.xaxis.set_major_locator(mdates.AutoDateLocator()) + plt.setp(ax.xaxis.get_majorticklabels(), rotation=30, ha="right") + + # Small table showing peak values + ax_tbl.axis("off") + table = ax_tbl.table( + cellText=[ + ["Disk Read (bi)", bi_peak_ts, f"{bi_peak_val:,.0f}"], + ["Disk Write (bo)", bo_peak_ts, f"{bo_peak_val:,.0f}"], + ["Disk Await (ms)", q_peak_ts, f"{q_peak_val:,.1f}"], + ], + colLabels=["Type", "Time", "Peak"], + cellColours=[["#dbe9f6"] * 3, ["#f6dbdb"] * 3, ["#dbf6db"] * 3], + colColours=["#cccccc"] * 3, + loc="center", + cellLoc="center", + ) + table.auto_set_font_size(False) + table.set_fontsize(9) + table.scale(0.4, 1.2) + + fig.text(0.99, 0.01, f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + ha="right", va="bottom", fontsize=7, color="gray") + + plt.tight_layout() + plt.savefig(args.output, dpi=150) + print(f"Saved chart to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/.claude/scripts/io-performance-analyzer/run_io_graph.sh b/.claude/scripts/io-performance-analyzer/run_io_graph.sh new file mode 100755 index 0000000000..e2adace3bb --- /dev/null +++ b/.claude/scripts/io-performance-analyzer/run_io_graph.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Build the PCP container and generate a Disk I/O performance graph. +# +# Usage: ./run_io_graph.sh [--timezone TZ] [--output-dir DIR] [pcp-data-dir] +# --timezone TZ : IANA timezone for timestamps (default: UTC) +# --output-dir DIR : directory for output CSV and PNG (default: script directory) +# pcp-data-dir : directory containing PCP archive files (default: auto-detect) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TIMEZONE="UTC" +OUTPUT_DIR="" + +# Parse options +while [[ $# -gt 0 ]]; do + case "$1" in + --timezone) + TIMEZONE="${2:?--timezone requires a value (e.g. UTC, US/Eastern)}" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="${2:?--output-dir requires a directory path}" + shift 2 + ;; + *) + DATA_DIR="$1" + shift + ;; + esac +done + +OUTPUT_DIR="${OUTPUT_DIR:-${SCRIPT_DIR}}" +mkdir -p "${OUTPUT_DIR}" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" + +DATA_DIR="${DATA_DIR:-$(find "${SCRIPT_DIR}" -name 'Latest' -exec dirname {} \; | head -1)}" + +if [[ -z "${DATA_DIR}" || ! -d "${DATA_DIR}" ]]; then + echo "ERROR: PCP data directory not found. Usage: $0 [--timezone TZ] [--output-dir DIR] " >&2 + exit 1 +fi + +DATA_DIR="$(cd "${DATA_DIR}" && pwd)" +IMAGE_NAME="pcp-io-graph" + +echo "==> Building container image..." +podman build -t "${IMAGE_NAME}" -f "${SCRIPT_DIR}/Dockerfile" "${SCRIPT_DIR}" + +echo "==> Extracting IO data and generating graph from ${DATA_DIR} (timezone: ${TIMEZONE})..." +podman run --rm \ + -v "${DATA_DIR}:/data/pcp:ro,Z" \ + -v "${SCRIPT_DIR}/extract_io.sh:/data/extract_io.sh:ro,Z" \ + -v "${SCRIPT_DIR}/parse_pcp.py:/data/parse_pcp.py:ro,Z" \ + -v "${SCRIPT_DIR}/plot_io.py:/data/plot_io.py:ro,Z" \ + -v "${OUTPUT_DIR}:/data/output:Z" \ + -e MPLCONFIGDIR=/tmp/matplotlib \ + "${IMAGE_NAME}" -c " + /data/extract_io.sh /data/pcp /data/output/io_data.json ${TIMEZONE} && \ + python3 /data/plot_io.py /data/output/io_data.json -o /data/output/disk_io_performance.png --timezone ${TIMEZONE} + " + +echo "==> Done!" +echo " JSON: ${OUTPUT_DIR}/io_data.json" +echo " Graph: ${OUTPUT_DIR}/disk_io_performance.png"