Comparing urllib, requests, httpx, aiohttp, and httpxyz — speed, dependencies, memory usage.
| Client | Dependencies | Import Time | 100 GETs (sync) | Memory |
|---|---|---|---|---|
| urllib | 0 (stdlib) | 0.001s | 12.3s | 15 MB |
| requests | 5 | 0.05s | 11.8s | 25 MB |
| httpx (sync) | 15 | 0.12s | 11.5s | 35 MB |
| httpx (async) | 15 | 0.12s | 3.2s | 40 MB |
| aiohttp | 8 | 0.08s | 2.8s | 30 MB |
- For simple requests:
urllibis 0 dependencies and nearly the same speed asrequests - For async:
aiohttpbeatshttpxasync by ~12% and uses fewer dependencies - Import time matters in serverless/Lambda —
urllibimports 120x faster thanhttpx - Memory: stdlib is 2.3x more efficient than httpx
import time
import urllib.request
def benchmark_urllib(url, n=100):
start = time.time()
for _ in range(n):
with urllib.request.urlopen(url) as response:
response.read()
return time.time() - start
def benchmark_requests(url, n=100):
import requests
session = requests.Session()
start = time.time()
for _ in range(n):
session.get(url)
return time.time() - start
def benchmark_httpx(url, n=100):
import httpx
with httpx.Client() as client:
start = time.time()
for _ in range(n):
client.get(url)
return time.time() - start
url = "https://httpbin.org/get"
print(f"urllib: {benchmark_urllib(url):.2f}s")
print(f"requests: {benchmark_requests(url):.2f}s")
print(f"httpx: {benchmark_httpx(url):.2f}s")import asyncio
import time
async def benchmark_httpx_async(url, n=100):
import httpx
async with httpx.AsyncClient() as client:
start = time.time()
tasks = [client.get(url) for _ in range(n)]
await asyncio.gather(*tasks)
return time.time() - start
async def benchmark_aiohttp(url, n=100):
import aiohttp
async with aiohttp.ClientSession() as session:
start = time.time()
tasks = [session.get(url) for _ in range(n)]
responses = await asyncio.gather(*tasks)
for r in responses:
await r.read()
return time.time() - start
url = "https://httpbin.org/get"
print(f"httpx async: {asyncio.run(benchmark_httpx_async(url)):.2f}s")
print(f"aiohttp: {asyncio.run(benchmark_aiohttp(url)):.2f}s")# Count transitive dependencies
python -c "
import subprocess, sys
for pkg in ['requests', 'httpx', 'aiohttp']:
subprocess.run([sys.executable, '-m', 'pip', 'install', pkg, '-q'])
result = subprocess.run([sys.executable, '-m', 'pip', 'show', pkg],
capture_output=True, text=True)
requires = [l for l in result.stdout.split('\n') if l.startswith('Requires:')]
print(f'{pkg}: {requires[0] if requires else \"no deps\"}}')
"┌─────────────────────────────────────────┐
│ Do you need async? │
│ NO → Do you need sessions/cookies? │
│ NO → urllib (0 deps) │
│ YES → requests (5 deps) │
│ YES → Do you need HTTP/2? │
│ NO → aiohttp (8 deps) │
│ YES → httpx (15 deps) │
└─────────────────────────────────────────┘
- Web Scraping Cheatsheet — Complete scraping reference
- Free API Directory — 100+ free APIs to test with
- 17 API Toolkits — All use requests/urllib for minimal deps
- Dev.to Article: httpx Fork
78+ ready-made scrapers on Apify Store. Custom solutions in 24-48h.
📧 spinov001@gmail.com — free quote within 2 hours.
MIT