Benchmark a simple BPE tokeniser (#339)

python · May 20, 2024 · 784d042 · 784d042
1 parent e094f2b
commit 784d042
Show file tree

Hide file tree

Showing 4 changed files with 312 additions and 0 deletions.
diff --git a/pyperformance/data-files/benchmarks/MANIFEST b/pyperformance/data-files/benchmarks/MANIFEST
@@ -22,6 +22,7 @@ async_tree_eager_memoization_tg	<local:async_tree>
 asyncio_tcp	<local>
 asyncio_tcp_ssl	<local:asyncio_tcp>
 asyncio_websockets	<local>
+bpe_tokeniser	<local>
 concurrent_imap	<local>
 coroutines	<local>
 coverage	<local>