From 0442986ca2d5a615d15501fe3694e7e6ed61778c Mon Sep 17 00:00:00 2001 From: Joseph Pollack Date: Thu, 13 Nov 2025 18:00:22 +0100 Subject: [PATCH] initial release --- .env.example | 334 ---- .github/workflows/build.yml | 4 +- .github/workflows/ci.yml | 4 +- .github/workflows/docs.yml | 4 + .github/workflows/publish-pypi.yml | 102 + .github/workflows/release.yml | 2 +- .readthedocs.yaml | 35 + CI_CD_IMPLEMENTATION_PLAN.md | 1711 ----------------- CI_CD_SETUP_STATUS.md | 129 -- DAEMON_IMPLEMENTATION_ASSESSMENT.md | 253 --- ccbt.toml | 582 +++--- ccbt.toml copy.example | 382 ---- ccbt/executor/manager.py | 1 + dev/requirements-rtd.txt | 27 + docs/pypi-setup.md | 212 ++ docs/readthedocs-setup.md | 156 ++ ...ck_throughput-20251113-165017-7cbe3b2.json | 53 + .../loopback_throughput_timeseries.json | 54 + env.example | 430 ++++- 19 files changed, 1410 insertions(+), 3065 deletions(-) delete mode 100644 .env.example create mode 100644 .github/workflows/publish-pypi.yml delete mode 100644 CI_CD_IMPLEMENTATION_PLAN.md delete mode 100644 CI_CD_SETUP_STATUS.md delete mode 100644 DAEMON_IMPLEMENTATION_ASSESSMENT.md delete mode 100644 ccbt.toml copy.example create mode 100644 dev/requirements-rtd.txt create mode 100644 docs/pypi-setup.md create mode 100644 docs/readthedocs-setup.md create mode 100644 docs/reports/benchmarks/runs/loopback_throughput-20251113-165017-7cbe3b2.json create mode 100644 docs/reports/benchmarks/timeseries/loopback_throughput_timeseries.json diff --git a/.env.example b/.env.example deleted file mode 100644 index aa9ae1c5..00000000 --- a/.env.example +++ /dev/null @@ -1,334 +0,0 @@ -# ccBitTorrent Environment Variables Configuration -# Copy this file to .env and modify values as needed -# All variables are prefixed with CCBT_ to avoid conflicts - -# ============================================================================= -# NETWORK CONFIGURATION -# ============================================================================= - -# Connection limits -CCBT_MAX_PEERS=200 # Maximum global peers (1-10000) -CCBT_MAX_PEERS_PER_TORRENT=50 # Maximum peers per torrent (1-1000) -CCBT_MAX_CONNECTIONS_PER_PEER=1 # Max parallel connections per peer (1-8) - -# Request pipeline settings -CCBT_PIPELINE_DEPTH=16 # Request pipeline depth (1-128) -CCBT_BLOCK_SIZE_KIB=16 # Block size in KiB (1-64) -CCBT_MIN_BLOCK_SIZE_KIB=4 # Minimum block size in KiB (1-64) -CCBT_MAX_BLOCK_SIZE_KIB=64 # Maximum block size in KiB (1-1024) - -# Socket tuning -CCBT_SOCKET_RCVBUF_KIB=256 # Socket receive buffer size in KiB (1-65536) -CCBT_SOCKET_SNDBUF_KIB=256 # Socket send buffer size in KiB (1-65536) -CCBT_TCP_NODELAY=true # Enable TCP_NODELAY (true/false) - -# Timeouts (seconds) -CCBT_CONNECTION_TIMEOUT=30.0 # Connection timeout (1.0-300.0) -CCBT_HANDSHAKE_TIMEOUT=10.0 # Handshake timeout (1.0-60.0) -CCBT_KEEP_ALIVE_INTERVAL=120.0 # Keep alive interval (30.0-600.0) -CCBT_PEER_TIMEOUT=60.0 # Peer inactivity timeout (5.0-600.0) -CCBT_DHT_TIMEOUT=2.0 # DHT request timeout (1.0-60.0) - -# Listen settings -CCBT_LISTEN_PORT=6881 # Listen port (1024-65535) -CCBT_LISTEN_INTERFACE=0.0.0.0 # Listen interface -CCBT_ENABLE_IPV6=true # Enable IPv6 support (true/false) - -# Transport protocols -CCBT_ENABLE_TCP=true # Enable TCP transport (true/false) -CCBT_ENABLE_UTP=false # Enable uTP transport (true/false) -CCBT_ENABLE_ENCRYPTION=false # Enable protocol encryption (true/false) - -# uTP (uTorrent Transport Protocol) Configuration (BEP 29) -CCBT_UTP_PREFER_OVER_TCP=true # Prefer uTP over TCP when both are supported (true/false) -CCBT_UTP_CONNECTION_TIMEOUT=30.0 # uTP connection timeout in seconds (5.0-300.0) -CCBT_UTP_MAX_WINDOW_SIZE=65535 # Maximum uTP receive window size in bytes (8192-65535) -CCBT_UTP_MTU=1200 # uTP MTU size (maximum UDP packet size) (576-65507) -CCBT_UTP_INITIAL_RATE=1500 # Initial send rate in bytes/second (1024-100000) -CCBT_UTP_MIN_RATE=512 # Minimum send rate in bytes/second (256-10000) -CCBT_UTP_MAX_RATE=1000000 # Maximum send rate in bytes/second (10000-10000000) -CCBT_UTP_ACK_INTERVAL=0.1 # ACK packet send interval in seconds (0.01-1.0) -CCBT_UTP_RETRANSMIT_TIMEOUT_FACTOR=4.0 # RTT multiplier for retransmit timeout (2.0-10.0) -CCBT_UTP_MAX_RETRANSMITS=10 # Maximum retransmission attempts before connection failure (3-50) - -# Choking strategy -CCBT_MAX_UPLOAD_SLOTS=4 # Maximum upload slots (1-20) -CCBT_OPTIMISTIC_UNCHOKE_INTERVAL=30.0 # Optimistic unchoke interval (1.0-600.0) -CCBT_UNCHOKE_INTERVAL=10.0 # Unchoke interval (1.0-600.0) - -# Rate limiting (KiB/s, 0 = unlimited) -CCBT_GLOBAL_DOWN_KIB=0 # Global download limit (0+) -CCBT_GLOBAL_UP_KIB=0 # Global upload limit (0+) -CCBT_PER_PEER_DOWN_KIB=0 # Per-peer download limit (0+) -CCBT_PER_PEER_UP_KIB=0 # Per-peer upload limit (0+) - -# Tracker settings -CCBT_TRACKER_TIMEOUT=30.0 # Tracker request timeout (5.0-120.0) -CCBT_TRACKER_CONNECT_TIMEOUT=10.0 # Tracker connection timeout (1.0-60.0) -CCBT_TRACKER_CONNECTION_LIMIT=50 # Maximum tracker connections (1-200) -CCBT_TRACKER_CONNECTIONS_PER_HOST=10 # Max connections per tracker host (1-50) -CCBT_DNS_CACHE_TTL=300 # DNS cache TTL in seconds (60-3600) - -# BitTorrent Protocol v2 (BEP 52) settings -CCBT_PROTOCOL_V2_ENABLE=true # Enable BitTorrent Protocol v2 support (BEP 52) (true/false) -CCBT_PROTOCOL_V2_PREFER=false # Prefer v2 protocol when both v1 and v2 are available (true/false) -CCBT_PROTOCOL_V2_SUPPORT_HYBRID=true # Support hybrid torrents (both v1 and v2 metadata) (true/false) -CCBT_PROTOCOL_V2_HANDSHAKE_TIMEOUT=30.0 # v2 handshake timeout in seconds (5.0-300.0) - -# ============================================================================= -# DISK CONFIGURATION -# ============================================================================= - -# Preallocation strategy: none, sparse, full, fallocate -CCBT_PREALLOCATE=full # Preallocation strategy -CCBT_SPARSE_FILES=false # Use sparse files if supported (true/false) - -# Write optimization -CCBT_WRITE_BATCH_KIB=64 # Write batch size in KiB (1-1024) -CCBT_WRITE_BUFFER_KIB=1024 # Write buffer size in KiB (0-65536) -CCBT_USE_MMAP=true # Use memory mapping (true/false) -CCBT_MMAP_CACHE_MB=128 # Memory-mapped cache size in MB (16-2048) -CCBT_MMAP_CACHE_CLEANUP_INTERVAL=30.0 # MMap cache cleanup interval (1.0-300.0) - -# Hash verification -CCBT_HASH_WORKERS=4 # Number of hash verification workers (1-32) -CCBT_HASH_CHUNK_SIZE=65536 # Chunk size for hash verification (1024-1048576) -CCBT_HASH_BATCH_SIZE=4 # Number of pieces to verify in parallel batches (1-64) -CCBT_HASH_QUEUE_SIZE=100 # Hash verification queue size (10-500) - -# I/O threading -CCBT_DISK_WORKERS=2 # Number of disk I/O workers (1-16) -CCBT_DISK_QUEUE_SIZE=200 # Disk I/O queue size (10-1000) -CCBT_CACHE_SIZE_MB=256 # Cache size in MB (16-4096) - -# Advanced settings -CCBT_DIRECT_IO=false # Use direct I/O (true/false) -CCBT_SYNC_WRITES=false # Synchronize writes (true/false) -CCBT_READ_AHEAD_KIB=64 # Read ahead size in KiB (0-1024) -CCBT_ENABLE_IO_URING=false # Enable io_uring on Linux if available (true/false) -CCBT_DOWNLOAD_PATH= # Default download path - -# Checkpoint settings -CCBT_CHECKPOINT_ENABLED=true # Enable download checkpointing (true/false) -CCBT_CHECKPOINT_FORMAT=both # Checkpoint file format (json/binary/both) -CCBT_CHECKPOINT_DIR= # Checkpoint directory (defaults to download_dir/.ccbt/checkpoints) -CCBT_CHECKPOINT_INTERVAL=30.0 # Checkpoint save interval in seconds (1.0-3600.0) -CCBT_CHECKPOINT_ON_PIECE=true # Save checkpoint after each verified piece (true/false) -CCBT_AUTO_RESUME=true # Automatically resume from checkpoint on startup (true/false) -CCBT_CHECKPOINT_COMPRESSION=true # Compress binary checkpoint files (true/false) -CCBT_AUTO_DELETE_CHECKPOINT_ON_COMPLETE=true # Auto-delete checkpoint when download completes (true/false) -CCBT_CHECKPOINT_RETENTION_DAYS=30 # Days to retain checkpoints before cleanup (1-365) - -# Fast Resume settings -CCBT_FAST_RESUME_ENABLED=true # Enable fast resume support (true/false) -CCBT_RESUME_SAVE_INTERVAL=30.0 # Interval to save resume data in seconds (1.0-3600.0) -CCBT_RESUME_VERIFY_ON_LOAD=true # Verify resume data integrity on load (true/false) -CCBT_RESUME_VERIFY_PIECES=10 # Number of pieces to verify on resume (0-100, 0 = disable) -CCBT_RESUME_DATA_FORMAT_VERSION=1 # Resume data format version (1-100) - -# BEP 47: File Attributes Configuration -CCBT_ATTRIBUTES_PRESERVE_ATTRIBUTES=true # Preserve file attributes (executable, hidden, symlinks) (true/false) -CCBT_ATTRIBUTES_SKIP_PADDING_FILES=true # Skip downloading padding files (BEP 47) (true/false) -CCBT_ATTRIBUTES_VERIFY_FILE_SHA1=false # Verify file SHA-1 hashes when provided (BEP 47) (true/false) -CCBT_ATTRIBUTES_APPLY_SYMLINKS=true # Create symlinks for files with attr='l' (true/false) -CCBT_ATTRIBUTES_APPLY_EXECUTABLE_BIT=true # Set executable bit for files with attr='x' (true/false) -CCBT_ATTRIBUTES_APPLY_HIDDEN_ATTR=true # Apply hidden attribute for files with attr='h' (Windows) (true/false) - -# ============================================================================= -# STRATEGY CONFIGURATION -# ============================================================================= - -# Piece selection strategy: round_robin, rarest_first, sequential -CCBT_PIECE_SELECTION=rarest_first # Piece selection strategy -CCBT_ENDGAME_DUPLICATES=2 # Endgame duplicate requests (1-10) -CCBT_ENDGAME_THRESHOLD=0.95 # Endgame mode threshold (0.1-1.0) -CCBT_STREAMING_MODE=false # Enable streaming mode (true/false) - -# Advanced strategy settings -CCBT_RAREST_FIRST_THRESHOLD=0.1 # Rarest first threshold (0.0-1.0) -CCBT_SEQUENTIAL_WINDOW=10 # Sequential window size (1-100) -CCBT_SEQUENTIAL_PRIORITY_FILES= # File paths to prioritize in sequential mode (comma-separated, optional) -CCBT_SEQUENTIAL_FALLBACK_THRESHOLD=0.1 # Fallback to rarest-first if availability < threshold (0.0-1.0) -CCBT_PIPELINE_CAPACITY=4 # Request pipeline capacity (1-32) - -# Piece priorities -CCBT_FIRST_PIECE_PRIORITY=true # Prioritize first piece (true/false) -CCBT_LAST_PIECE_PRIORITY=false # Prioritize last piece (true/false) - -# ============================================================================= -# DISCOVERY CONFIGURATION -# ============================================================================= - -# DHT settings -CCBT_ENABLE_DHT=true # Enable DHT (true/false) -CCBT_DHT_PORT=6882 # DHT port (1024-65535) -# DHT bootstrap nodes (comma-separated) -CCBT_DHT_BOOTSTRAP_NODES=router.bittorrent.com:6881,dht.transmissionbt.com:6881,router.utorrent.com:6881,dht.libtorrent.org:25401 - -# BEP 32: IPv6 Extension for DHT -CCBT_DHT_ENABLE_IPV6=true # Enable IPv6 DHT support (BEP 32) (true/false) -CCBT_DHT_PREFER_IPV6=true # Prefer IPv6 addresses over IPv4 when available (true/false) -# IPv6 DHT bootstrap nodes (comma-separated, format: [hostname:port or [IPv6]:port]) -CCBT_DHT_IPV6_BOOTSTRAP_NODES= - -# BEP 43: Read-only DHT Nodes -CCBT_DHT_READONLY_MODE=false # Enable read-only DHT mode (BEP 43) (true/false) - -# BEP 45: Multiple-Address Operation for DHT -CCBT_DHT_ENABLE_MULTIADDRESS=true # Enable multi-address support (BEP 45) (true/false) -CCBT_DHT_MAX_ADDRESSES_PER_NODE=4 # Maximum addresses to track per node (BEP 45) (1-16) - -# BEP 44: Storing Arbitrary Data in the DHT -CCBT_DHT_ENABLE_STORAGE=false # Enable DHT storage (BEP 44) (true/false) -CCBT_DHT_STORAGE_TTL=3600 # Storage TTL in seconds (BEP 44) (60-86400) -CCBT_DHT_MAX_STORAGE_SIZE=1000 # Maximum storage value size in bytes (BEP 44) (100-10000) - -# BEP 51: DHT Infohash Indexing -CCBT_DHT_ENABLE_INDEXING=true # Enable infohash indexing (BEP 51) (true/false) -CCBT_DHT_INDEX_SAMPLES_PER_KEY=8 # Maximum samples per index key (BEP 51) (1-100) - -# PEX settings -CCBT_ENABLE_PEX=true # Enable Peer Exchange (true/false) -CCBT_PEX_INTERVAL=30.0 # Peer Exchange announce interval in seconds (5.0-3600.0) - -# Tracker settings -CCBT_ENABLE_HTTP_TRACKERS=true # Enable HTTP trackers (true/false) -CCBT_ENABLE_UDP_TRACKERS=true # Enable UDP trackers (true/false) -CCBT_TRACKER_ANNOUNCE_INTERVAL=1800.0 # Tracker announce interval in seconds (60.0-86400.0) -CCBT_TRACKER_SCRAPE_INTERVAL=3600.0 # Tracker scrape interval in seconds (60.0-86400.0) -CCBT_TRACKER_AUTO_SCRAPE=false # Automatically scrape trackers when adding torrents (true/false) - -# Private torrent settings (BEP 27) -CCBT_STRICT_PRIVATE_MODE=true # Enforce strict BEP 27 rules for private torrents (true/false) - -# ============================================================================= -# OBSERVABILITY CONFIGURATION -# ============================================================================= - -# Logging -CCBT_LOG_LEVEL=INFO # Log level (DEBUG/INFO/WARNING/ERROR/CRITICAL) -CCBT_LOG_FILE= # Log file path (empty = stdout) -CCBT_LOG_FORMAT=%(asctime)s - %(name)s - %(levelname)s - %(message)s # Log format string -CCBT_STRUCTURED_LOGGING=true # Use structured logging (true/false) -CCBT_LOG_CORRELATION_ID=true # Include correlation IDs (true/false) - -# Metrics -CCBT_ENABLE_METRICS=true # Enable metrics collection (true/false) -CCBT_METRICS_PORT=9090 # Metrics port (1024-65535) -CCBT_METRICS_INTERVAL=5.0 # Metrics collection interval in seconds (0.5-3600.0) - -# Tracing -CCBT_ENABLE_PEER_TRACING=false # Enable peer tracing (true/false) -CCBT_TRACE_FILE= # Path to write traces (empty = disabled) -CCBT_ALERTS_RULES_PATH=.ccbt/alerts.json # Path to alert rules JSON file - -# ============================================================================= -# LIMITS CONFIGURATION -# ============================================================================= - -# Global rate limits (KiB/s, 0 = unlimited) -CCBT_LIMITS_GLOBAL_DOWN_KIB=0 # Global download limit (0+) -CCBT_LIMITS_GLOBAL_UP_KIB=0 # Global upload limit (0+) - -# Per-torrent rate limits (KiB/s, 0 = unlimited) -CCBT_LIMITS_PER_TORRENT_DOWN_KIB=0 # Per-torrent download limit (0+) -CCBT_LIMITS_PER_TORRENT_UP_KIB=0 # Per-torrent upload limit (0+) - -# Per-peer rate limits (KiB/s, 0 = unlimited) -CCBT_LIMITS_PER_PEER_UP_KIB=0 # Per-peer upload limit (0+) - -# Scheduler settings -CCBT_SCHEDULER_SLICE_MS=100 # Scheduler time slice in ms (1-1000) - -# ============================================================================= -# SECURITY CONFIGURATION -# ============================================================================= - -CCBT_ENABLE_ENCRYPTION=false # Enable protocol encryption (true/false) -CCBT_ENCRYPTION_MODE=preferred # Encryption mode: disabled/preferred/required -CCBT_ENCRYPTION_DH_KEY_SIZE=768 # DH key size in bits: 768 or 1024 -CCBT_ENCRYPTION_PREFER_RC4=true # Prefer RC4 cipher for compatibility (true/false) -CCBT_ENCRYPTION_ALLOWED_CIPHERS=rc4,aes # Allowed ciphers (comma-separated: rc4,aes,chacha20) -CCBT_ENCRYPTION_ALLOW_PLAIN_FALLBACK=true # Allow fallback to plain connection (true/false) -CCBT_VALIDATE_PEERS=true # Validate peers before exchanging data (true/false) -CCBT_RATE_LIMIT_ENABLED=true # Enable security rate limiter (true/false) -CCBT_MAX_CONNECTIONS_PER_PEER=1 # Maximum parallel connections per peer (1-8) - -# IP Filter settings -CCBT_ENABLE_IP_FILTER=false # Enable IP filtering (true/false) -CCBT_FILTER_MODE=block # Filter mode: block or allow -CCBT_FILTER_FILES= # Comma-separated filter file paths -CCBT_FILTER_URLS= # Comma-separated filter list URLs -CCBT_FILTER_UPDATE_INTERVAL=86400.0 # Update interval in seconds (3600.0-604800.0) -CCBT_FILTER_CACHE_DIR=~/.ccbt/filters # Filter cache directory -CCBT_FILTER_LOG_BLOCKED=true # Log blocked connections (true/false) - -# ============================================================================= -# SSL/TLS CONFIGURATION -# ============================================================================= - -# SSL/TLS settings -CCBT_ENABLE_SSL_TRACKERS=true # Enable SSL for tracker connections (true/false) -CCBT_ENABLE_SSL_PEERS=false # Enable SSL for peer connections (true/false) -CCBT_SSL_VERIFY_CERTIFICATES=true # Verify SSL certificates (true/false) -CCBT_SSL_CA_CERTIFICATES= # Path to CA certificates file or directory -CCBT_SSL_CLIENT_CERTIFICATE= # Path to client certificate file (PEM format) -CCBT_SSL_CLIENT_KEY= # Path to client private key file (PEM format) -CCBT_SSL_PROTOCOL_VERSION=TLSv1.2 # TLS protocol version (TLSv1.2, TLSv1.3, PROTOCOL_TLS) -CCBT_SSL_ALLOW_INSECURE_PEERS=true # Allow insecure peers for opportunistic encryption (true/false) - -# ============================================================================= -# PROXY CONFIGURATION -# ============================================================================= - -CCBT_PROXY_ENABLE_PROXY=false # Enable proxy support (true/false) -CCBT_PROXY_TYPE=http # Proxy type: http/socks4/socks5 -CCBT_PROXY_HOST= # Proxy server hostname or IP -CCBT_PROXY_PORT= # Proxy server port (1-65535) -CCBT_PROXY_USERNAME= # Proxy username for authentication -CCBT_PROXY_PASSWORD= # Proxy password (encrypted in storage) -CCBT_PROXY_FOR_TRACKERS=true # Use proxy for tracker requests (true/false) -CCBT_PROXY_FOR_PEERS=false # Use proxy for peer connections (true/false) -CCBT_PROXY_FOR_WEBSEEDS=true # Use proxy for WebSeed requests (true/false) -CCBT_PROXY_BYPASS_LIST= # Comma-separated list of hosts/IPs to bypass proxy - -# ============================================================================= -# MACHINE LEARNING CONFIGURATION -# ============================================================================= - -CCBT_ML_PEER_SELECTION_ENABLED=false # Enable ML-based peer selection (true/false) -CCBT_ML_PIECE_PREDICTION_ENABLED=false # Enable ML piece prediction (true/false) - -# ============================================================================= -# CLI-SPECIFIC OVERRIDES -# ============================================================================= -# These can be used to override any configuration value via environment variables -# Format: CCBT_
_