From c3ca34c7e9a8dac64a608c0daaabc2995e9196f1 Mon Sep 17 00:00:00 2001 From: perpil <97474956+perpil@users.noreply.github.com> Date: Fri, 29 May 2026 10:47:29 -0700 Subject: [PATCH 1/2] Add historical graph generation workflow + Add a GitHub Actions workflow to generate and commit SVG performance graphs when data changes. + Introduce graph-generator/app.py to parse data JSONs, produce 2y and 180d SVG charts and commit them to /graphs. + Inject a small clickable graph thumbnail into index.html, add CSS for graph container and thumbnail styling, and update script.js to set thumbnail and large-graph URLs (handles GitHub Pages vs local). + Uses pandas/matplotlib to generate graphs + Gemini CLI w/model 3.0 Flash was used to create this commit --- .github/workflows/generate-graphs.yml | 46 +++++ docs/css/custom.css | 14 +- docs/index.html | 7 +- docs/script.js | 21 +++ graph-generator/app.py | 256 ++++++++++++++++++++++++++ 5 files changed, 341 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/generate-graphs.yml create mode 100644 graph-generator/app.py diff --git a/.github/workflows/generate-graphs.yml b/.github/workflows/generate-graphs.yml new file mode 100644 index 0000000000..a8c0534e09 --- /dev/null +++ b/.github/workflows/generate-graphs.yml @@ -0,0 +1,46 @@ +name: Generate Graphs + +on: + push: + branches: + - main + paths: + - "data/last.json" + workflow_dispatch: + +permissions: + contents: write + +jobs: + generate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pandas matplotlib + + - name: Generate graphs + run: python graph-generator/app.py + + - name: Commit and push graphs + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git add graphs/*.svg + if git diff --staged --quiet; then + echo "No changes to graphs" + else + git commit -m "docs: update performance graphs" + git push + fi diff --git a/docs/css/custom.css b/docs/css/custom.css index 80ddbdcd69..7665777a25 100644 --- a/docs/css/custom.css +++ b/docs/css/custom.css @@ -3,7 +3,7 @@ html { min-height: 100%; } body { - margin-bottom: 120px; + margin-bottom: 120px; } hr { margin-bottom: 2px; @@ -13,7 +13,7 @@ hr { position: absolute; bottom: 0; width: 100%; - height: 100px; + height: 100px; line-height: 60px; background-color: #f5f5f5; text-align: center; @@ -66,6 +66,16 @@ hr { text-align: right; font-size: 9px; } +.graph-container { + cursor: pointer; + transition: transform 0.2s; +} +.graph-container:hover { + transform: scale(1.02); +} +.smallGraph { + border-radius: 4px; +} .badge:empty { display: inline-block; } diff --git a/docs/index.html b/docs/index.html index 943acc811a..2feba0d412 100644 --- a/docs/index.html +++ b/docs/index.html @@ -74,6 +74,11 @@

Lambda Cold Starts benchmark
by runtime name +
+ + Performance history + +
❄ 💾 @@ -93,4 +98,4 @@

Lambda Cold Starts benchmark
by = date_2y] + if not group_2y.empty: + futures.append( + executor.submit( + save_plot, + group_2y, + runtime, + arch, + pkg, + mem_limit, + region, + "2y", + (1024, 768), + ) + ) + + # Small Graph (180 days) + group_180d = group[group["generatedAt"] >= date_180d] + if not group_180d.empty: + futures.append( + executor.submit( + save_plot, + group_180d, + runtime, + arch, + pkg, + mem_limit, + region, + "180d", + (300, 170), + is_small=True, + ) + ) + + for future in concurrent.futures.as_completed(futures): + try: + future.result() + except Exception as e: + print(f"A task failed: {e}") + + +def save_plot( + data, runtime, arch, pkg, mem_limit, region, suffix, dimensions, is_small=False +): + width, height = dimensions + dpi = 100 + + # Background color for the web page + bg_color = "#212529" + + fig, ax1 = plt.subplots(figsize=(width / dpi, height / dpi), dpi=dpi) + + if is_small: + fig.patch.set_facecolor(bg_color) + ax1.set_facecolor(bg_color) + + # Line colors (keep blue/red but ensure they are vibrant) + ax1.plot( + data["generatedAt"], + data["acd"], + color="#339af0", + label="Cold Start", + linewidth=1.2, + ) + ax1.plot( + data["generatedAt"], + data["ad"], + color="#ff6b6b", + label="Warm Start", + linewidth=1.2, + ) + + # Axis and label styling for dark background + ax1.xaxis.set_major_formatter(mdates.DateFormatter("%m/%Y")) + ax1.xaxis.set_major_locator(mdates.MonthLocator(interval=2)) + + ax1.tick_params(axis="both", which="major", labelsize=7, colors="#dee2e6") + ax1.spines["bottom"].set_color("#495057") + ax1.spines["top"].set_visible(False) + ax1.spines["right"].set_visible(False) + ax1.spines["left"].set_color("#495057") + + ax1.set_ylabel("ms", fontsize=7, color="#dee2e6") + ax1.grid(True, linestyle="--", alpha=0.1, color="#f8f9fa") + plt.tight_layout(pad=0.5) + else: + # Large graph keeps default white background for "click-in" view unless requested otherwise + # Plot Cold Start (Blue) and Warm Start (Red) on ax1 + ax1.plot( + data["generatedAt"], + data["acd"], + color="#007bff", + label="Cold Start", + linewidth=1.5, + marker="o", + markersize=4, + ) + ax1.plot( + data["generatedAt"], + data["ad"], + color="#dc3545", + label="Warm Start", + linewidth=1.5, + marker="o", + markersize=4, + ) + + # Create twin axis for Memory Usage (Orange) only on large graphs + ax2 = ax1.twinx() + ax2.plot( + data["generatedAt"], + data["mu"], + color="#ff7f0e", + label="Memory", + linewidth=1.5, + linestyle="--", + ) + + ax1.xaxis.set_major_formatter(mdates.DateFormatter("%m/%Y")) + ax1.xaxis.set_major_locator(mdates.MonthLocator(interval=3)) + + ax1.set_xlabel("Time") + ax1.set_ylabel("Duration [ms]") + ax2.set_ylabel("Memory Usage [MB]", color="#ff7f0e") + ax1.tick_params(axis="y", labelcolor="#333") + ax2.tick_params(axis="y", labelcolor="#ff7f0e") + + plt.title(f"{runtime} ({arch}, {pkg}, {mem_limit}MB, {region})") + ax1.grid(True, linestyle="--", alpha=0.6) + + # Legend: Combine legends from both axes + lines1, labels1 = ax1.get_legend_handles_labels() + lines2, labels2 = ax2.get_legend_handles_labels() + ax1.legend( + lines1 + lines2, labels1 + labels2, loc="upper left", fontsize="small" + ) + + plt.xticks(rotation=45) + plt.tight_layout() + + # Format memory as integer for the filename + mem_int = int(float(mem_limit)) + filename = f"last-{runtime}-{arch}-{pkg}-{mem_int}-{region}-{suffix}.svg" + filepath = os.path.join(GRAPHS_DIR, filename) + plt.savefig(filepath) + plt.close(fig) + + +if __name__ == "__main__": + print("Loading data...") + df = load_data() + if df.empty: + print("No data found.") + else: + print(f"Processing {len(df)} data points...") + generate_graphs(df) + print("Done.") From 647cab3016340db2346bd86ad41aa96ce0c12e56 Mon Sep 17 00:00:00 2001 From: perpil <97474956+perpil@users.noreply.github.com> Date: Fri, 29 May 2026 13:40:15 -0700 Subject: [PATCH 2/2] Update permissions and readme --- README.md | 6 +++++- graph-generator/app.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) mode change 100644 => 100755 graph-generator/app.py diff --git a/README.md b/README.md index a1dddd87cb..8a84d424e7 100644 --- a/README.md +++ b/README.md @@ -82,10 +82,14 @@ After all these invocations, all information stored in DynamoDB is aggregated an ### Step 4 -A static website, hosted on GitHub pages here: https://maxday.github.io/lambda-perf/ fetches this JSON file and displays the result in a (nice?) UI. +Using the data from the last 2 years, SVG charts are generated to show how the performance of each runtime has evolved over time. These files are stored in the /graphs folder. ### Step 5 +A static website, hosted on GitHub pages here: https://maxday.github.io/lambda-perf/ fetches this JSON file and displays the result in a (nice?) UI. + +### Step 6 + Hack/Fork/Send PR and create your own benchmarks! ## Disclaimer diff --git a/graph-generator/app.py b/graph-generator/app.py old mode 100644 new mode 100755 index 83b92780bd..48e2a952fb --- a/graph-generator/app.py +++ b/graph-generator/app.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import concurrent.futures import glob import json