Export historical size metrics to Datadog (AI-5520)

.github/workflows/measure-disk-usage.yml

@@ -29,97 +29,80 @@ jobs:
         ddev config set repo core
     - name: Measure disk usage (uncompressed)
       run: | 
-        mkdir -p status_visualizations
-        ddev size status --csv > size-uncompressed.csv
-        ddev size status  --save_to_png_path status_visualizations/uncompressed.png > size-uncompressed.txt
+        ddev size status  >> size-uncompressed.txt --to-dd-key ${{secrets.DD_API_KEY}}
+        ddev size status  --format png,csv,markdown
         cat size-uncompressed.txt
         echo "# Size (uncompressed)" >> $GITHUB_STEP_SUMMARY
-        echo '```' >> $GITHUB_STEP_SUMMARY
-        cat size-uncompressed.txt >> $GITHUB_STEP_SUMMARY
-        echo '```' >> $GITHUB_STEP_SUMMARY
+        cat uncompressed_status.md >> $GITHUB_STEP_SUMMARY
 
     - name: Measure disk usage (compressed)
       run: |
-        mkdir -p status_visualizations
-        ddev size status --csv --compressed > size-compressed.csv
-        ddev size status --compressed  --save_to_png_path status_visualizations/compressed.png > size-compressed.txt
+        ddev size status --compressed  > size-compressed.txt --to-dd-key ${{secrets.DD_API_KEY}}
+        ddev size status --compressed  --format png,csv,markdown
         cat size-compressed.txt
         echo "# Size (compressed)" >> $GITHUB_STEP_SUMMARY
-        echo '```' >> $GITHUB_STEP_SUMMARY
-        cat size-compressed.txt >> $GITHUB_STEP_SUMMARY
-        echo '```' >> $GITHUB_STEP_SUMMARY
+        cat compressed_status.md >> $GITHUB_STEP_SUMMARY
 
 
     - name: Measure disk usage differences from last commit (uncompressed)
-      if: false  # Disabled for now: size difference can be misleading due to dependencies not being built in the same PR
       run: | 
-        mkdir -p diff_visualizations
         BEFORE=$(git rev-parse HEAD^)
         AFTER=$(git rev-parse HEAD)
-        ddev size diff $BEFORE $AFTER --csv > diff-uncompressed.csv
-        ddev size diff $BEFORE $AFTER  --save_to_png_path diff_visualizations/diff-uncompressed-linux.png > diff-uncompressed.txt
+        ddev size diff $BEFORE $AFTER > diff-uncompressed.txt
+        ddev size diff $BEFORE $AFTER  --format png,csv,markdown
         cat diff-uncompressed.txt
         echo "# Size diff (uncompressed)" >> $GITHUB_STEP_SUMMARY
-        echo '```' >> $GITHUB_STEP_SUMMARY
-        cat diff-uncompressed.txt >> $GITHUB_STEP_SUMMARY
-        echo '```' >> $GITHUB_STEP_SUMMARY
+        cat uncompressed_diff.md >> $GITHUB_STEP_SUMMARY
 
     - name: Measure disk usage differences from last commit (compressed)
-      if: false  # Disabled for now: size difference can be misleading due to dependencies not being built in the same PR
       run: | 
-        mkdir -p diff_visualizations
         BEFORE=$(git rev-parse HEAD^)
         AFTER=$(git rev-parse HEAD)
-        ddev size diff $BEFORE $AFTER --compressed --csv > diff-compressed.csv
-        ddev size diff $BEFORE $AFTER --compressed  --save_to_png_path diff_visualizations/diff-compressed-linux.png > diff-compressed.txt
+        ddev size diff $BEFORE $AFTER --compressed > diff-compressed.txt
+        ddev size diff $BEFORE $AFTER --compressed  --format png,csv,markdown
         cat diff-compressed.txt
         echo "# Size diff (compressed)" >> $GITHUB_STEP_SUMMARY
-        echo '```' >> $GITHUB_STEP_SUMMARY
-        cat diff-compressed.txt >> $GITHUB_STEP_SUMMARY
-        echo '```' >> $GITHUB_STEP_SUMMARY
+        cat compressed_diff.md >> $GITHUB_STEP_SUMMARY
 
     - name: Upload file sizes (uncompressed)
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
       with:
-        name: size-uncompressed.csv
-        path: size-uncompressed.csv
+        name: uncompressed_status.csv
+        path: uncompressed_status.csv
         if-no-files-found: error
 
     - name: Upload file sizes (compressed)
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
       with:
-        name: size-compressed.csv
-        path: size-compressed.csv
+        name: compressed_status.csv
+        path: compressed_status.csv
         if-no-files-found: error
 
     - name: Upload file sizes diff (uncompressed)
-      if: false # Disabled for now: size difference can be misleading due to dependencies not being built in the same PR
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
       with:
-        name: diff-uncompressed.csv
-        path: diff-uncompressed.csv
+        name: uncompressed_diff.csv
+        path: uncompressed_diff.csv
         if-no-files-found: error     
 
     - name: Upload file sizes diff (compressed)
-      if: false  # Disabled for now: size difference can be misleading due to dependencies not being built in the same PR
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
       with:
-        name: diff-compressed.csv
-        path: diff-compressed.csv
+        name: compressed_diff.csv
+        path: compressed_diff.csv
         if-no-files-found: error
 
     - name: Upload status PNGs
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
       with:
         name: size-visuals
-        path: status_visualizations/
+        path: size_status_visualizations/
         if-no-files-found: error
 
     - name: Upload diff PNGs
-      if: false # Disabled for now: size difference can be misleading due to dependencies not being built in the same PR
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
       with:
         name: diff-visuals
-        path: diff_visualizations/
+        path: size_diff_visualizations/
         if-no-files-found: error
 

ddev/changelog.d/20431.added

@@ -0,0 +1,2 @@
+- Adds the required logic to upload historical size metrics to a specified Datadog organization.
+- Updates the CI pipeline to send metrics to Datadog on pushes to the master branch. Note that the metrics may not be fully accurate yet, as dependency sizes could be outdated since the lockfile updates are handled in a separate PR.

ddev/src/ddev/cli/size/create_dashboard.py

@@ -1,11 +1,11 @@
 import json
-from typing import Any
+import os
 
 import click
 import requests
 
 from ddev.cli.application import Application
-from ddev.cli.size.utils.common_funcs import get_org, get_valid_platforms
+from ddev.cli.size.utils.common_funcs import get_org
 
 
 @click.command()
@@ -27,23 +27,20 @@ def create_dashboard(
     """
     try:
         config_file_info = get_org(app, dd_org)
-        if 'api_key' not in config_file_info:
+        if "api_key" not in config_file_info:
             raise RuntimeError("No API key found in config file")
-        if 'app_key' not in config_file_info:
+        if "app_key" not in config_file_info:
             raise RuntimeError("No APP key found in config file")
-        if 'site' not in config_file_info:
+        if "site" not in config_file_info:
             raise RuntimeError("No site found in config file")
         headers = {
             "DD-API-KEY": config_file_info["api_key"],
             "DD-APPLICATION-KEY": config_file_info["app_key"],
             "Content-Type": "application/json",
         }
-
-        payload = {
-            "title": "Disk Usage Status for Integrations and Dependencies",
-            "layout_type": "ordered",
-            "widgets": create_json(app),
-        }
+        json_path = os.path.join(os.path.dirname(__file__), "utils", "dashboard.json")
+        with open(json_path, "r", encoding="utf-8") as f:
+            payload = json.load(f)
 
         response = requests.post(
             f"https://api.{config_file_info['site']}/api/v1/dashboard",
@@ -57,88 +54,3 @@ def create_dashboard(
         print(f"Dashboard URL: https://app.{config_file_info['site']}{resp_json['url']}")
     except Exception as e:
         app.abort(str(e))
-
-
-def create_json(app: Application) -> list[dict[str, Any]]:
-    valid_platforms = get_valid_platforms(app.repo.path)
-    widgets: list[dict[str, Any]] = []
-
-    for size_type in ["compressed", "uncompressed"]:
-        for platform in valid_platforms:
-            # Treemap widget
-            widgets.append(
-                {
-                    "definition": {
-                        "type": "treemap",
-                        "title": f"{size_type.capitalize()} sizes in {platform}",
-                        "requests": [
-                            {
-                                "queries": [
-                                    {
-                                        "data_source": "metrics",
-                                        "name": "query2",
-                                        "query": f"avg:datadog.agent_integrations.size_analyzer.{size_type}"
-                                        f"{{platform:{platform}}} by {{name_type,name}}",
-                                        "aggregator": "last",
-                                    }
-                                ],
-                                "response_format": "scalar",
-                                "style": {"palette": "classic"},
-                                "formulas": [
-                                    {
-                                        "formula": "query2",
-                                        "number_format": {
-                                            "unit": {
-                                                "type": "canonical_unit",
-                                                "unit_name": "byte_in_binary_bytes_family",
-                                            }
-                                        },
-                                    }
-                                ],
-                            }
-                        ],
-                    }
-                }
-            )
-            # Timeseries widget
-            widgets.append(
-                {
-                    "definition": {
-                        "title": f"Timeline of {size_type} sizes in {platform}",
-                        "type": "timeseries",
-                        "requests": [
-                            {
-                                "response_format": "timeseries",
-                                "queries": [
-                                    {
-                                        "name": "query1",
-                                        "data_source": "metrics",
-                                        "query": f"sum:datadog.agent_integrations.size_analyzer.{size_type}"
-                                        f"{{platform:{platform}}}",
-                                    }
-                                ],
-                                "formulas": [
-                                    {
-                                        "formula": "query1",
-                                        "number_format": {
-                                            "unit": {
-                                                "type": "canonical_unit",
-                                                "unit_name": "byte_in_binary_bytes_family",
-                                            }
-                                        },
-                                    }
-                                ],
-                                "style": {
-                                    "palette": "dog_classic",
-                                    "order_by": "values",
-                                    "line_type": "solid",
-                                    "line_width": "normal",
-                                },
-                                "display_type": "line",
-                            }
-                        ],
-                    }
-                }
-            )
-
-    return widgets

ddev/src/ddev/cli/size/diff.py

@@ -2,6 +2,7 @@
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
 
+import os
 from datetime import datetime
 from typing import Optional
 
@@ -87,8 +88,8 @@ def diff(
                 date = datetime.strptime(date_str, "%b %d %Y").date()
                 if date < MINIMUM_DATE:
                     raise ValueError(f"First commit must be after {MINIMUM_DATE.strftime('%b %d %Y')} ")
-                valid_platforms = get_valid_platforms(gitRepo.repo_dir)
                 valid_versions = get_valid_versions(gitRepo.repo_dir)
+                valid_platforms = get_valid_platforms(gitRepo.repo_dir, valid_versions)
                 if platform and platform not in valid_platforms:
                     raise ValueError(f"Invalid platform: {platform}")
                 elif version and version not in valid_versions:
@@ -139,7 +140,7 @@ def diff_mode(
     dependencies = get_diff(dependencies_b, dependencies_a, "Dependency")
 
     if integrations + dependencies == []:
-        params["app"].display_error(
+        params["app"].display(
             f"No size differences were detected between the selected commits for {params['platform']}"
         )
         return []
@@ -151,16 +152,15 @@ def diff_mode(
                 module["Size"] = f"+{module['Size']}"
 
     if not params["format"] or params["format"] == ["png"]:  # if no format is provided for the data print the table
-        print_table(params["app"], "Status", formatted_modules)
+        print_table(params["app"], "Diff", formatted_modules)
 
-    treemap_path = (
-        f"treemap_{params['platform']}_{params['version']}.png"
-        if params["format"] and "png" in params["format"]
-        else None
-    )
+    treemap_path = None
+    if params["format"] and "png" in params["format"]:
+        treemap_path = os.path.join("size_diff_visualizations", f"treemap_{params['platform']}_{params['version']}.png")
 
     if params["show_gui"] or treemap_path:
         plot_treemap(
+            params["app"],
             formatted_modules,
             f"Disk Usage Differences for {params['platform']} and Python version {params['version']}",
             params["show_gui"],

ddev/src/ddev/cli/size/historical_metrics.py

@@ -0,0 +1,79 @@
+import argparse
+import subprocess
+from datetime import datetime
+from pathlib import Path
+
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, TextColumn
+from utils.common_funcs import GitRepo
+
+console = Console()
+
+
+def upload_historical_metrics(date_from_str: str, org: str) -> None:
+    current_path = Path(__file__).resolve()
+    repo_path = current_path.parents[5]
+
+    date_from = datetime.strptime(date_from_str, "%Y-%m-%d")
+    min_date = datetime.strptime("2024-09-18", "%Y-%m-%d")
+
+    if date_from < min_date:
+        raise ValueError(f"Date ({date_from}) must be after 2024-09-18")
+    try:
+        console.print(f"[green]Processing repository: {repo_path}")
+        with GitRepo(repo_path) as gitRepo:
+            commits = gitRepo._run(f"git log --pretty=format:%H --since='{date_from}'")
+            console.print(f"Found {len(commits)} commits to process")
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                transient=True,
+                console=console,
+            ) as progress:
+                commit_task = progress.add_task("[cyan]Processing commits...")
+
+                for i, commit in enumerate(commits, 1):
+                    date, _, _ = gitRepo.get_commit_metadata(commit)
+                    progress.update(
+                        commit_task, description=f"Processing commit {i}/{len(commits)}: {commit[:8]} ({date})"
+                    )
+                    print(f"Processing commit {i}/{len(commits)}: {commit[:8]} ({date})", flush=True)
+                    gitRepo.checkout_commit(commit)
+                    result = subprocess.run(
+                        ["ddev", "--here", "size", "status", "--to-dd-org", org],
+                        cwd=gitRepo.repo_dir,
+                        text=True,
+                        capture_output=True,
+                    )
+                    if result.returncode != 0:
+                        console.print(f"[red]Error in commit {commit}: {result.stderr}")
+                        continue
+
+                    result = subprocess.run(
+                        ["ddev", "--here", "size", "status", "--compressed", "--to-dd-org", org],
+                        text=True,
+                        cwd=gitRepo.repo_dir,
+                        capture_output=True,
+                    )
+
+                    if result.returncode != 0:
+                        console.print(f"[red]Error in commit {commit}: {result.stderr}")
+                        continue
+
+                    progress.advance(commit_task)
+                progress.update(commit_task, description="[green]All commits processed!")
+
+    except KeyboardInterrupt:
+        console.print("[red]Process interrupted by user.")
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(description='Upload historical metrics to Datadog')
+    parser.add_argument('--date-from', help='Start date in YYYY-MM-DD format')
+    parser.add_argument('--org', default='default', help='Organization name')
+
+    args = parser.parse_args()
+    date_from = args.date_from
+    org = args.org
+    upload_historical_metrics(date_from, org)