veb: add zstd compression support (#25816)

Author: davlgd

vlib/veb/README.md

@@ -419,22 +419,24 @@ app.static_mime_types['.what'] = 'txt/plain'
 app.handle_static('static', true)!
 ```
 
-### Gzip compression for static files
+### Compression for static files (zstd/gzip)
 
-veb provides automatic gzip compression for static files with smart caching. When enabled,
-veb will serve compressed versions of your static files to clients that support gzip encoding,
-reducing bandwidth usage and improving load times.
+veb provides automatic compression (zstd and gzip) for static files with smart caching.
+When enabled, veb will serve compressed versions of your static files to clients that
+support compression, reducing bandwidth usage and improving load times. Zstd is preferred
+over gzip when the client supports both.
 
 **How it works:**
 
-1. **Manual pre-compression**: If you create `.gz` files manually, veb will serve them in
-   zero-copy streaming mode for maximum performance.
+1. **Manual pre-compression**: If you create `.zst` or `.gz` files manually, veb will serve
+   them in zero-copy streaming mode for maximum performance.
 2. **Lazy compression cache**: Files smaller than the threshold are automatically compressed
-   on first request and cached as `.gz` files on disk.
-3. **Cache validation**: If the original file is modified, the `.gz` cache is automatically
-   regenerated on the next request.
+   on first request and cached as `.zst` or `.gz` files on disk (zstd preferred when client
+   supports it).
+3. **Cache validation**: If the original file is modified, the compressed cache is
+   automatically regenerated on the next request.
 4. **Streaming for large files**: Files larger than the threshold are served uncompressed in
-   streaming mode (unless a manual `.gz` file exists).
+   streaming mode (unless a manual `.zst` or `.gz` file exists).
 
 **Example:**
 
@@ -453,21 +455,26 @@ pub struct App {
 }
 
 pub fn (mut app App) index(mut ctx Context) veb.Result {
-	return ctx.html('<h1>Gzip compression demo</h1><p>Visit <a href="/app.js">/app.js</a> or <a href="/style.css">/style.css</a></p>')
+	return ctx.html('<h1>Compression demo</h1>
+    <p>Visit <a href="/app.js">/app.js</a> or <a href="/style.css">/style.css</a>
+    </p>')
 }
 
 fn main() {
 	mut app := &App{}
 
-	// Enable static file gzip compression (disabled by default)
-	app.enable_static_gzip = true
-	app.static_gzip_max_size = 524288 // Maximum file size for auto-compression is 512 KB (default: 1MB)
+	// Enable static file compression (zstd/gzip, disabled by default)
+	// Use enable_static_zstd and enable_static_gzip for specific compression
+	app.enable_static_compression = true
+	app.static_compression_max_size = 524288 // Maximum file size for auto-compression is 512 KB (default: 1MB)
 
 	// Serve files from the 'static' directory
 	app.handle_static('static', true)!
 
-	// Add the gzip middleware to compress dynamic routes as well
-	app.use(veb.encode_gzip[Context]())
+	// Add the content encoding middleware to compress dynamic routes as well
+	// This will use zstd if the client supports it, otherwise gzip
+	// Use encode_gzip or encode_zstd for specific compression
+	app.use(veb.encode_auto[Context]())
 
 	veb.run[App, Context](mut app, 8080)
 }
@@ -480,59 +487,68 @@ Create test files in the `static` directory:
 mkdir -p static
 echo "console.log('Hello from V web!');" > static/app.js
 echo "body { margin: 0; }" > static/style.css
-# Pre-compress style.css manually for zero-copy streaming
-gzip -k static/style.css
+# Pre-compress style.css manually for zero-copy streaming (zstd or gzip)
+zstd -k static/style.css  # creates style.css.zst
+# or: gzip -k static/style.css  # creates style.css.gz
 ```
 
 Run the server, it will listen on port 8080:
 ```bash
 v run server.v
 ```
 
-Test gzip compression with cURL:
+Test compression with cURL:
 ```bash
-# Test lazy compression cache - app.js will be compressed on first request
-curl -H "Accept-Encoding: gzip" -i http://localhost:8080/app.js
+# Test zstd compression (preferred when client supports it)
+curl -H "Accept-Encoding: zstd, gzip" -i http://localhost:8080/app.js
+# Expected headers:
+# Content-Encoding: zstd
+# Vary: Accept-Encoding
 
+# Test gzip fallback (when client doesn't support zstd)
+curl -H "Accept-Encoding: gzip" -i http://localhost:8080/app.js
 # Expected headers:
 # Content-Encoding: gzip
 # Vary: Accept-Encoding
 
 # Request with automatic decompression
-curl -H "Accept-Encoding: gzip" --compressed http://localhost:8080/app.js
+curl -H "Accept-Encoding: zstd, gzip" --compressed http://localhost:8080/app.js
 
-# Request without gzip encoding - should return uncompressed content
+# Request without encoding - should return uncompressed content
 curl -i http://localhost:8080/app.js
 
-# Verify that .gz cache file was created
-ls -lh static/app.js.gz
+# Verify that compressed cache file was created
+ls -lh static/app.js.zst static/app.js.gz 2>/dev/null
 
-# Test manual pre-compression - style.css.gz is served directly (zero-copy)
-# The pre-compressed .gz file is served without loading into memory
-curl -H "Accept-Encoding: gzip" -i http://localhost:8080/style.css
+# Test manual pre-compression - style.css.zst is served directly (zero-copy)
+curl -H "Accept-Encoding: zstd" -i http://localhost:8080/style.css
 ```
 
 **Performance tips:**
 
-- For production, you can pre-compress your static files (e.g., `gzip -k static/app.js`)
-  and veb will serve them directly without loading into memory.
+- For production, you can pre-compress your static files with zstd (`zstd -k static/app.js`)
+  or gzip (`gzip -k static/app.js`) and veb will serve them directly without loading into memory.
+- Zstd offers better compression ratio and speed than gzip - use it when possible.
+
+**Priority order**: When both `.zst` and `.gz` files exist for the same source file, veb will
+serve `.zst` if the client supports zstd, otherwise `.gz` if gzip is supported.
+
 - The lazy cache is created on first request, so the first visitor pays a small
   compression cost, but all subsequent requests are served at zero-copy speed.
-- Large files (> threshold) are always streamed, ensuring low memory usage even for
-  large assets.
-- The `encode_gzip` middleware compresses dynamic routes and small files loaded in
-  memory (takeover mode).
-- If `.gz` caching fails (e.g., on read-only filesystems), veb automatically falls
-  back to serving compressed content from memory. You can  set `static_gzip_max_size = 0`
+- Large files (> threshold) are always streamed, ensuring low memory usage even for large assets.
+- The `encode_auto` middleware automatically chooses zstd or gzip based on client support. You can
+  also use `encode_zstd` or `encode_gzip` for specific compression.
+- If caching fails (e.g., on read-only filesystems), veb automatically falls
+  back to serving compressed content from memory. You can set `static_compression_max_size = 0`
   to disable auto-compression completely. For optimal performance on read-only systems,
-  pre-compress all files with `gzip -k`.
+  pre-compress all files with `zstd -k` or `gzip -k`.
 
 ### Markdown content negotiation
 
 veb can provide automatic content negotiation for markdown files, allowing you to serve
 markdown content when the client explicitly requests it via the `Accept` header.
-This is compliant to [llms.txt](https://llmstxt.org/) proposal and useful for documentations that can serve
-the same content in multiple formats, more efficiently to AI services using it.
+This is compliant to [llms.txt](https://llmstxt.org/) proposal and useful for documentations that
+can serve the same content in multiple formats, more efficiently to AI services using it.
 
 **How it works:**
 

vlib/veb/consts.v

@@ -142,6 +142,7 @@ pub const mime_types = {
 	'.xml':    'application/xml'
 	'.xul':    'application/vnd.mozilla.xul+xml'
 	'.zip':    'application/zip'
+	'.zst':    'application/zstd'
 	'.3gp':    'video/3gpp'
 	'.3g2':    'video/3gpp2'
 	'.7z':     'application/x-7z-compressed'

vlib/veb/context.v

@@ -1,6 +1,7 @@
 module veb
 
 import compress.gzip
+import compress.zstd
 import json
 import net
 import net.http
@@ -32,15 +33,17 @@ mut:
 	done bool
 	// If the `Connection: close` header is present the connection should always be closed
 	client_wants_to_close bool
-	// Configuration for static file gzip compression (set by serve_if_static)
-	enable_static_gzip   bool
-	static_gzip_max_size int
+	// Configuration for static file compression (set by serve_if_static)
+	enable_static_gzip          bool
+	enable_static_zstd          bool
+	enable_static_compression   bool
+	static_compression_max_size int
 	// if true the response should not be sent and the connection should be closed
 	// manually.
 	takeover    bool
 	return_file string
-	// already_compressed indicates that the response body is already gzip-compressed
-	// and the encode_gzip middleware should skip it
+	// already_compressed indicates that the response body is already compressed (zstd/gzip)
+	// and the compression middlewares should skip it
 	already_compressed bool
 pub:
 	// TODO: move this to `handle_request`
@@ -207,80 +210,64 @@ fn (mut ctx Context) send_file(content_type string, file_path string) Result {
 	}
 	file.close()
 
-	// Check if client accepts gzip encoding
+	// Check which encodings the client accepts
 	accept_encoding := ctx.req.header.get(.accept_encoding) or { '' }
+	client_accepts_zstd := accept_encoding.contains('zstd')
 	client_accepts_gzip := accept_encoding.contains('gzip')
 
-	max_size_bytes := ctx.static_gzip_max_size
+	max_size_bytes := ctx.static_compression_max_size
 
-	// Try to serve pre-compressed .gz file if static gzip is enabled and client accepts it
-	if ctx.enable_static_gzip && client_accepts_gzip {
-		gz_path := '${file_path}.gz'
+	// Determine which compression modes are enabled
+	use_zstd := (ctx.enable_static_zstd && client_accepts_zstd)
+		|| (ctx.enable_static_compression && client_accepts_zstd)
+	use_gzip := (ctx.enable_static_gzip && client_accepts_gzip)
+		|| (ctx.enable_static_compression && client_accepts_gzip)
 
-		// Check if .gz file exists and is up-to-date (newer or same age as original)
-		if os.exists(gz_path) {
-			gz_mtime := os.file_last_mod_unix(gz_path)
-			orig_mtime := os.file_last_mod_unix(file_path)
+	// Try to serve pre-compressed files if any compression is enabled
+	if use_zstd || use_gzip {
+		orig_mtime := os.file_last_mod_unix(file_path)
 
-			if gz_mtime >= orig_mtime {
-				// Serve existing .gz file in streaming mode (zero-copy)
-				ctx.return_type = .file
-				ctx.return_file = gz_path
-				ctx.res.header.set(.content_encoding, 'gzip')
-				ctx.res.header.set(.vary, 'Accept-Encoding')
-
-				// Get .gz file size for Content-Length header
-				gz_size := os.file_size(gz_path)
-				ctx.res.header.set(.content_length, gz_size.str())
-
-				ctx.send_response_to_client(content_type, '')
-				ctx.already_compressed = true
+		// Try zstd first if enabled (better compression), then gzip
+		if use_zstd {
+			if ctx.serve_precompressed_file(content_type, file_path, '.zst', 'zstd', orig_mtime) {
+				return Result{}
+			}
+		}
+		if use_gzip {
+			if ctx.serve_precompressed_file(content_type, file_path, '.gz', 'gzip', orig_mtime) {
 				return Result{}
 			}
 		}
 
-		// .gz doesn't exist or is outdated: create it if file is small enough
+		// No pre-compressed file available: create one if file is small enough
 		if file_size < max_size_bytes {
 			// Load, compress, save, and serve
 			data := os.read_file(file_path) or {
 				eprintln('[veb] error while trying to read file: ${err.msg()}')
 				return ctx.server_error('could not read resource')
 			}
 
-			compressed := gzip.compress(data.bytes()) or {
-				// Fallback: serve uncompressed in streaming mode
-				ctx.return_type = .file
-				ctx.return_file = file_path
-				ctx.res.header.set(.content_length, file_size.str())
-				ctx.send_response_to_client(content_type, '')
-				return Result{}
+			// Try zstd first if enabled, then gzip
+			if use_zstd {
+				if result := ctx.serve_compressed_static(content_type, file_path, data,
+					.zstd)
+				{
+					return result
+				}
 			}
-
-			// Try to save compressed version for future requests
-			mut write_success := true
-			os.write_file(gz_path, compressed.bytestr()) or {
-				eprintln('[veb] warning: could not save .gz file (readonly filesystem?): ${err.msg()}')
-				write_success = false
+			if use_gzip {
+				if result := ctx.serve_compressed_static(content_type, file_path, data,
+					.gzip)
+				{
+					return result
+				}
 			}
 
-			if write_success {
-				// Serve the newly cached .gz file in streaming mode (zero-copy)
-				ctx.return_type = .file
-				ctx.return_file = gz_path
-				ctx.res.header.set(.content_encoding, 'gzip')
-				ctx.res.header.set(.vary, 'Accept-Encoding')
-				ctx.res.header.set(.content_length, compressed.len.str())
-				ctx.send_response_to_client(content_type, '')
-				ctx.already_compressed = true
-			} else {
-				// Fallback: serve compressed content from memory (no caching)
-				// This happens on readonly filesystems or when write permissions are missing
-				ctx.res.header.set(.content_encoding, 'gzip')
-				ctx.res.header.set(.vary, 'Accept-Encoding')
-				ctx.send_response_to_client(content_type, compressed.bytestr())
-				ctx.already_compressed = true
-			}
-			return Result{}
+			// Compression failed: serve uncompressed in streaming mode
+			ctx.return_type = .file
+			ctx.return_file = file_path
+			ctx.res.header.set(.content_length, file_size.str())
+			return ctx.send_response_to_client(content_type, '')
 		}
 	}
 
@@ -301,6 +288,71 @@ fn (mut ctx Context) send_file(content_type string, file_path string) Result {
 	return Result{}
 }
 
+// serve_precompressed_file serves an existing pre-compressed file (.zst or .gz) if it exists and is fresh.
+// Returns true if the file was served, false otherwise.
+fn (mut ctx Context) serve_precompressed_file(content_type string, file_path string, ext string, encoding_name string, orig_mtime i64) bool {
+	compressed_path := '${file_path}${ext}'
+	if !os.exists(compressed_path) {
+		return false
+	}
+	compressed_mtime := os.file_last_mod_unix(compressed_path)
+	if compressed_mtime < orig_mtime {
+		return false
+	}
+	// Serve existing compressed file in streaming mode (zero-copy)
+	ctx.return_type = .file
+	ctx.return_file = compressed_path
+	ctx.res.header.set(.content_encoding, encoding_name)
+	ctx.res.header.set(.vary, 'Accept-Encoding')
+	compressed_size := os.file_size(compressed_path)
+	ctx.res.header.set(.content_length, compressed_size.str())
+	ctx.send_response_to_client(content_type, '')
+	ctx.already_compressed = true
+	return true
+}
+
+// serve_compressed_static compresses data and serves it, optionally caching to disk.
+// Returns Result on success, none on compression failure.
+fn (mut ctx Context) serve_compressed_static(content_type string, file_path string, data string, encoding ContentEncoding) ?Result {
+	compressed, ext, encoding_name := match encoding {
+		.zstd {
+			c := zstd.compress(data.bytes()) or { return none }
+			c, '.zst', 'zstd'
+		}
+		.gzip {
+			c := gzip.compress(data.bytes()) or { return none }
+			c, '.gz', 'gzip'
+		}
+	}
+
+	compressed_path := '${file_path}${ext}'
+
+	// Try to save compressed version for future requests
+	mut write_success := true
+	os.write_file(compressed_path, compressed.bytestr()) or {
+		eprintln('[veb] warning: could not save ${ext} file (readonly filesystem?): ${err.msg()}')
+		write_success = false
+	}
+
+	if write_success {
+		// Serve the newly cached file in streaming mode (zero-copy)
+		ctx.return_type = .file
+		ctx.return_file = compressed_path
+		ctx.res.header.set(.content_encoding, encoding_name)
+		ctx.res.header.set(.vary, 'Accept-Encoding')
+		ctx.res.header.set(.content_length, compressed.len.str())
+		ctx.send_response_to_client(content_type, '')
+		ctx.already_compressed = true
+	} else {
+		// Fallback: serve compressed content from memory (no caching)
+		ctx.res.header.set(.content_encoding, encoding_name)
+		ctx.res.header.set(.vary, 'Accept-Encoding')
+		ctx.send_response_to_client(content_type, compressed.bytestr())
+		ctx.already_compressed = true
+	}
+	return Result{}
+}
+
 // Response HTTP_OK with s as payload
 pub fn (mut ctx Context) ok(s string) Result {
 	mut mime := if ctx.content_type.len == 0 { 'text/plain' } else { ctx.content_type }