Add eval scoring stack (jiwer/pyannote) with lazy imports

Formula/assembly.rb

@@ -69,6 +69,11 @@
     sha256 "44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"
   end
 
+  resource "click" do
+    url "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz"
+    sha256 "ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"
+  end
+
   resource "colorama" do
     url "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz"
     sha256 "08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"
@@ -134,6 +139,16 @@
     sha256 "4251acc80e2b7c9b7b8823456ea0fceeb0734dac2df7636d3c711b38476b5a76"
   end
 
+  resource "jiwer" do
+    url "https://files.pythonhosted.org/packages/12/1e/963dfc249d5bbe88079b57a22556e981ddd9208e4b6116e48bdbfc01f26b/jiwer-4.0.0.tar.gz"
+    sha256 "ae9c051469102a61ef0927100baeeb4546f78d180c9b0948281d08eaf44c191e"
+  end
+
+  resource "joblib" do
+    url "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz"
+    sha256 "8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"
+  end
+
   resource "keyring" do
     url "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz"
     sha256 "fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b"
@@ -154,11 +169,31 @@
     sha256 "48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d"
   end
 
+  resource "narwhals" do
+    url "https://files.pythonhosted.org/packages/62/3c/c4ef2164a71c1a63d7f1ae411c4082c5fa872405106db60a4b7114989ad7/narwhals-2.22.1.tar.gz"
+    sha256 "d62920805a0a43b7ff8b54b0c0d3142d796f8a9301836ada37e573d6a33cbcd9"
+  end
+
+  resource "numpy" do
+    url "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz"
+    sha256 "f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda"
+  end
+
   resource "openai" do
     url "https://files.pythonhosted.org/packages/3c/a6/5815fe2e2aca74b36c650d1bd43b69827cee568073d0d2d9b6fc5aaac80c/openai-2.41.0.tar.gz"
     sha256 "db5c362acd6604b84f076abbefa66826ea4b46ecba2954ed866e6a149a1352c0"
   end
 
+  resource "packaging" do
+    url "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz"
+    sha256 "ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"
+  end
+
+  resource "pandas" do
+    url "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz"
+    sha256 "696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"
+  end
+
   resource "platformdirs" do
     url "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz"
     sha256 "31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"
@@ -169,6 +204,21 @@
     sha256 "28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855"
   end
 
+  resource "pyannote-core" do
+    url "https://files.pythonhosted.org/packages/a3/be/4a35ea31c685aef801f7f35c193e7766ca1bb948ae497a625cbfaa8c31ba/pyannote_core-6.0.1.tar.gz"
+    sha256 "4b4ada3276f6df4e073fa79166636e3597d0dcb5a0fe26014a3477867cc033fb"
+  end
+
+  resource "pyannote-database" do
+    url "https://files.pythonhosted.org/packages/65/45/6210274c187cc457e854be8b56c6819fa14376f27e7e2b6021b2aa02449a/pyannote_database-6.1.1.tar.gz"
+    sha256 "bbe76da738257a9e64061123d9694ad7e949c4f171d91a9269606d873528cd10"
+  end
+
+  resource "pyannote-metrics" do
+    url "https://files.pythonhosted.org/packages/74/ba/7dbc2f790d5e321e46dc1e250ff00b69cdefc0c5695b811e96a4b932cddd/pyannote_metrics-4.1.tar.gz"
+    sha256 "afe24c54ee0799e8cfbe8ee85fa517793c3450bb7eae8fedd1a77ccec0343f7e"
+  end
+
   resource "pycparser" do
     url "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz"
     sha256 "600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"
@@ -194,36 +244,76 @@
     sha256 "6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"
   end
 
+  resource "python-dateutil" do
+    url "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz"
+    sha256 "37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"
+  end
+
   resource "python-dotenv" do
     url "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz"
     sha256 "2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3"
   end
 
+  resource "pyyaml" do
+    url "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz"
+    sha256 "d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"
+  end
+
   resource "questionary" do
     url "https://files.pythonhosted.org/packages/f6/45/eafb0bba0f9988f6a2520f9ca2df2c82ddfa8d67c95d6625452e97b204a5/questionary-2.1.1.tar.gz"
     sha256 "3d7e980292bb0107abaa79c68dd3eee3c561b83a0f89ae482860b181c8bd412d"
   end
 
+  resource "rapidfuzz" do
+    url "https://files.pythonhosted.org/packages/2c/21/ef6157213316e85790041254259907eb722e00b03480256c0545d98acd33/rapidfuzz-3.14.5.tar.gz"
+    sha256 "ba10ac57884ce82112f7ed910b67e7fb6072d8ef2c06e30dc63c0f604a112e0e"
+  end
+
   resource "rich" do
     url "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz"
     sha256 "edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36"
   end
 
+  resource "scikit-learn" do
+    url "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz"
+    sha256 "8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557"
+  end
+
+  resource "scipy" do
+    url "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz"
+    sha256 "95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0"
+  end
+
   resource "shellingham" do
     url "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz"
     sha256 "8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"
   end
 
+  resource "six" do
+    url "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz"
+    sha256 "ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"
+  end
+
   resource "sniffio" do
     url "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz"
     sha256 "f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"
   end
 
+  resource "sortedcontainers" do
+    url "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz"
+    sha256 "25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"
+  end
+
   resource "sounddevice" do
     url "https://files.pythonhosted.org/packages/2a/f9/2592608737553638fca98e21e54bfec40bf577bb98a61b2770c912aab25e/sounddevice-0.5.5.tar.gz"
     sha256 "22487b65198cb5bf2208755105b524f78ad173e5ab6b445bdab1c989f6698df3"
   end
 
+  resource "threadpoolctl" do
+    url "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz"
+    sha256 "8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e"
+  end
+
   resource "tomli-w" do
     url "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz"
     sha256 "2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021"
@@ -249,6 +339,11 @@
     sha256 "ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464"
   end
 
+  resource "tzdata" do
+    url "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz"
+    sha256 "9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10"
+  end
+
   resource "wcwidth" do
     url "https://files.pythonhosted.org/packages/2c/ee/afaf0f85a9a18fe47a67f1e4422ed6cf1fe642f0ae0a2f81166231303c52/wcwidth-0.7.0.tar.gz"
     sha256 "90e3a7ea092341c44b99562e75d09e4d5160fe7a3974c6fb842a101a95e7eed0"
@@ -270,5 +365,11 @@
 
   test do
     assert_match version.to_s, shell_output("#{bin}/assembly --version")
+    # --help imports the full command tree, so a resource missing from this
+    # formula fails here instead of on the user's first real command.
+    assert_match "transcribe", shell_output("#{bin}/assembly --help")
+    # The `assembly eval` scoring stack (jiwer/pyannote) is imported lazily, so
+    # --help can't catch it being absent — probe it directly.
+    system libexec/"bin/python", "-c", "import jiwer, pyannote.metrics"
   end
 end

README.md

@@ -17,15 +17,21 @@ The AssemblyAI CLI (`assembly`) brings speech AI to your terminal: transcribe fi
 
 ## 📦 Installation
 
+Requires Python 3.12+ (Homebrew brings its own; for pipx/uv see the `--python` hint below).
+
+> ⚠️ The `assemblyai-cli` package on PyPI is **not** this project — install with one of the
+> commands below, not `pip install assemblyai-cli`.
+
 ### Homebrew (recommended — macOS / Linux)
 
 ```sh
 brew tap assemblyai/cli https://github.com/AssemblyAI/cli
-brew trust assemblyai/cli
+brew trust assemblyai/cli   # only needed when HOMEBREW_REQUIRE_TAP_TRUST is set; harmless otherwise
 brew install assembly
 ```
 
-Homebrew pulls in `ffmpeg` and `portaudio`, so `transcribe`, `stream`, and `agent` work out of the box.
+Homebrew pulls in `ffmpeg` and `portaudio`, so `stream` and `agent` work out of the box.
+Plain `transcribe` uploads your file directly and needs neither.
 
 ### pipx / uv
 
@@ -41,7 +47,13 @@ Or with uv:
 uv tool install "git+https://github.com/AssemblyAI/cli.git"
 ```
 
-Requires Python 3.12+. On Linux, install PortAudio once for microphone support (`sudo apt-get install libportaudio2`), and have [`ffmpeg`](https://ffmpeg.org) on `PATH` to decode non-WAV audio.
+If your default interpreter is older than Python 3.12, add `--python python3.12` (pipx) or
+`--python 3.12` (uv) to the install command.
+
+Only `stream` and `agent` need extras: on Linux, install PortAudio once for microphone support
+(Debian/Ubuntu: `sudo apt-get install libportaudio2`; Fedora: `sudo dnf install portaudio`), and
+have [`ffmpeg`](https://ffmpeg.org) on `PATH` to stream non-WAV audio. Plain `transcribe` needs
+neither.
 
 ## 📋 Key Features
 
@@ -57,6 +69,9 @@ Requires Python 3.12+. On Linux, install PortAudio once for microphone support (
 
 ## 🔐 Authentication
 
+New to AssemblyAI? Create a free account at
+[assemblyai.com/dashboard](https://www.assemblyai.com/dashboard) to get an API key.
+
 ### Option 1: Browser login (recommended)
 
 ```sh
@@ -95,7 +110,13 @@ Then your own audio:
 assembly transcribe call.mp3
 ```
 
-Stream the microphone live (Ctrl-C to stop):
+Stream the hosted sample live (no microphone needed):
+
+```sh
+assembly stream --sample
+```
+
+Or stream your microphone (Ctrl-C to stop):
 
 ```sh
 assembly stream

aai_cli/commands/doctor.py

@@ -114,7 +114,7 @@ def _check_api_key(profile: str) -> Check:
     return _check(
         "api-key",
         "fail",
-        "API key was rejected (HTTP 401).",
+        "API key was rejected (HTTP 401/403).",
         fix="Run 'assembly login' with a valid key.",
         affects=["everything"],
     )
@@ -135,7 +135,10 @@ def check_ffmpeg() -> Check:
             "transcription (including YouTube) works without it, as does streaming a "
             "16 kHz mono WAV."
         ),
-        fix="Install ffmpeg (macOS: brew install ffmpeg; Debian/Ubuntu: apt-get install ffmpeg).",
+        fix=(
+            "Install ffmpeg (macOS: brew install ffmpeg; Debian/Ubuntu: apt-get install "
+            "ffmpeg; Fedora: dnf install ffmpeg; Windows: winget install ffmpeg)."
+        ),
         affects=["stream/agent (non-WAV file or URL input)"],
     )
 
@@ -176,7 +179,10 @@ def check_audio() -> Check:
             "audio",
             "warn",
             f"audio system unavailable: {exc}",
-            fix="On Linux install PortAudio: sudo apt-get install libportaudio2",
+            fix=(
+                "Install PortAudio (Debian/Ubuntu: sudo apt-get install libportaudio2; "
+                "Fedora: sudo dnf install portaudio; macOS: brew install portaudio)."
+            ),
             affects=affects,
         )
     if inputs == 0:

aai_cli/commands/login.py

@@ -49,7 +49,7 @@ def body(state: AppState, json_mode: bool) -> None:
             # obtained, so account self-service commands won't work for this profile.
             if not client.validate_key(api_key):
                 raise APIError(
-                    "That API key was rejected (HTTP 401).",
+                    "That API key was rejected (HTTP 401/403).",
                     suggestion="Check the key and retry.",
                 )
             config.set_api_key(profile, api_key)

aai_cli/update_check.py

@@ -44,10 +44,12 @@ def detect_upgrade_command() -> str:
     exe = (sys.executable or "").lower()
     if "/cellar/" in exe or "/homebrew/" in exe or exe.startswith("/usr/local/"):
         return "brew upgrade assembly"
+    # pipx/uv track installs by *distribution* name (aai-cli), not the console
+    # command (assembly) — "pipx upgrade assembly" fails with "not installed".
     if "pipx" in exe:
-        return "pipx upgrade assembly"
+        return "pipx upgrade aai-cli"
     if "/uv/tools/" in exe:
-        return "uv tool upgrade assembly"
+        return "uv tool upgrade aai-cli"
     return ""
 
 

aai_cli/wer.py

@@ -4,26 +4,36 @@
 the alignment math is never re-derived here. Texts are normalized the way ASR
 benchmarks conventionally are (lowercase, punctuation stripped, whitespace
 collapsed) so a transcript isn't penalized for casing or punctuation style.
-No SDK, no Rich: the command layer owns all rendering.
+jiwer is imported lazily inside the scoring functions (mirroring ``der.py``'s
+lazy pyannote import) so this module stays import-cheap for the command layer —
+an install that's missing the eval scoring stack must still run every other
+command. No SDK, no Rich: the command layer owns all rendering.
 """
 
 from __future__ import annotations
 
 from dataclasses import dataclass
+from typing import TYPE_CHECKING
 
-import jiwer
 from pydantic import TypeAdapter
 
-# The normalization applied to both reference and hypothesis before alignment.
-_TRANSFORM = jiwer.Compose(
-    [
-        jiwer.ToLowerCase(),
-        jiwer.RemovePunctuation(),
-        jiwer.RemoveMultipleSpaces(),
-        jiwer.Strip(),
-        jiwer.ReduceToListOfListOfWords(),
-    ]
-)
+if TYPE_CHECKING:
+    import jiwer
+
+
+def _transform() -> jiwer.Compose:
+    """The normalization applied to both reference and hypothesis before alignment."""
+    import jiwer
+
+    return jiwer.Compose(
+        [
+            jiwer.ToLowerCase(),
+            jiwer.RemovePunctuation(),
+            jiwer.RemoveMultipleSpaces(),
+            jiwer.Strip(),
+            jiwer.ReduceToListOfListOfWords(),
+        ]
+    )
 
 
 # jiwer's transform output is only partially typed; validate the shape instead
@@ -37,7 +47,7 @@ def normalize_words(text: str) -> list[str]:
     Exposed so the dataset loader can reject a reference that normalizes to
     nothing (e.g. punctuation-only) with the same rule the scorer uses.
     """
-    return _SENTENCES.validate_python(_TRANSFORM(text))[0]
+    return _SENTENCES.validate_python(_transform()(text))[0]
 
 
 @dataclass(frozen=True)
@@ -59,11 +69,14 @@ def score(reference: str, hypothesis: str) -> Score:
     dataset loader rejects empty-reference rows), so ``Score.wer`` is always
     well-defined.
     """
+    import jiwer
+
+    transform = _transform()
     out = jiwer.process_words(
         reference,
         hypothesis,
-        reference_transform=_TRANSFORM,
-        hypothesis_transform=_TRANSFORM,
+        reference_transform=transform,
+        hypothesis_transform=transform,
     )
     return Score(
         errors=out.substitutions + out.deletions + out.insertions,