all: support $embed_file('embed.vv', .zlib) (#12654)

Author: Le0Developer

cmd/tools/vcompress.v

@@ -0,0 +1,44 @@
+module main
+
+import compress.zlib
+import os
+
+enum CompressionType {
+	zlib
+}
+
+fn main() {
+	if os.args.len != 5 {
+		eprintln('v compress <type> <in> <out>')
+		eprintln('supported types: zlib')
+		exit(1)
+	}
+	compression_type := match os.args[2] {
+		'zlib' {
+			CompressionType.zlib
+		}
+		else {
+			eprintln('unsupported type: ${os.args[1]}')
+			exit(1)
+		}
+	}
+	path := os.args[3]
+	content := os.read_bytes(path) or {
+		eprintln('unable to read "$path": $err')
+		exit(1)
+	}
+	compressed := match compression_type {
+		.zlib {
+			zlib.compress(content) or {
+				eprintln('compression error: $err')
+				exit(1)
+			}
+		}
+	}
+	out_path := os.args[4]
+
+	os.write_file_array(out_path, compressed) or {
+		eprintln('failed to write "$out_path": $err')
+		exit(1)
+	}
+}

cmd/v/v.v

@@ -20,6 +20,7 @@ const (
 		'build-vbinaries',
 		'check-md',
 		'complete',
+		'compress',
 		'doc',
 		'doctor',
 		'fmt',

doc/docs.md

@@ -4982,6 +4982,17 @@ executable, increasing your binary size, but making it more self contained
 and thus easier to distribute. In this case, `embedded_file.data()` will cause *no IO*,
 and it will always return the same data.
 
+`$embed_file` supports compression of the embedded file when compiling with `-prod`.
+Currently only one compression type is supported: `zlib`
+
+```v ignore
+import os
+fn main() {
+	embedded_file := $embed_file('v.png', .zlib) // compressed using zlib
+	os.write_file('exported.png', embedded_file.to_string()) ?
+}
+```
+
 #### `$tmpl` for embedding and parsing V template files
 
 V has a simple template language for text and html templates, and they can easily

vlib/v/ast/ast.v

@@ -646,8 +646,14 @@ pub mut:
 
 pub struct EmbeddedFile {
 pub:
-	rpath string // used in the source code, as an ID/key to the embed
-	apath string // absolute path during compilation to the resource
+	rpath            string // used in the source code, as an ID/key to the embed
+	apath            string // absolute path during compilation to the resource
+	compression_type string
+pub mut:
+	// these are set by gen_embed_file_init in v/gen/c/embed
+	is_compressed bool
+	bytes         []byte
+	len           int
 }
 
 // Each V source file is represented by one File structure.
@@ -1542,8 +1548,7 @@ pub:
 	is_vweb   bool
 	vweb_tmpl File
 	//
-	is_embed   bool
-	embed_file EmbeddedFile
+	is_embed bool
 	//
 	is_env  bool
 	env_pos token.Position
@@ -1554,6 +1559,7 @@ pub mut:
 	result_type Type
 	env_value   string
 	args        []CallArg
+	embed_file  EmbeddedFile
 }
 
 pub struct None {

vlib/v/checker/checker.v

@@ -20,20 +20,21 @@ const int_min = int(0x80000000)
 const int_max = int(0x7FFFFFFF)
 
 const (
-	valid_comptime_if_os            = ['windows', 'ios', 'macos', 'mach', 'darwin', 'hpux', 'gnu',
+	valid_comptime_if_os             = ['windows', 'ios', 'macos', 'mach', 'darwin', 'hpux', 'gnu',
 		'qnx', 'linux', 'freebsd', 'openbsd', 'netbsd', 'bsd', 'dragonfly', 'android', 'solaris',
 		'haiku', 'serenity', 'vinix']
-	valid_comptime_if_compilers     = ['gcc', 'tinyc', 'clang', 'mingw', 'msvc', 'cplusplus']
-	valid_comptime_if_platforms     = ['amd64', 'i386', 'aarch64', 'arm64', 'arm32', 'rv64', 'rv32']
-	valid_comptime_if_cpu_features  = ['x64', 'x32', 'little_endian', 'big_endian']
-	valid_comptime_if_other         = ['js', 'debug', 'prod', 'test', 'glibc', 'prealloc',
+	valid_comptime_compression_types = ['none', 'zlib']
+	valid_comptime_if_compilers      = ['gcc', 'tinyc', 'clang', 'mingw', 'msvc', 'cplusplus']
+	valid_comptime_if_platforms      = ['amd64', 'i386', 'aarch64', 'arm64', 'arm32', 'rv64', 'rv32']
+	valid_comptime_if_cpu_features   = ['x64', 'x32', 'little_endian', 'big_endian']
+	valid_comptime_if_other          = ['js', 'debug', 'prod', 'test', 'glibc', 'prealloc',
 		'no_bounds_checking', 'freestanding', 'threads', 'js_node', 'js_browser', 'js_freestanding']
-	valid_comptime_not_user_defined = all_valid_comptime_idents()
-	array_builtin_methods           = ['filter', 'clone', 'repeat', 'reverse', 'map', 'slice',
+	valid_comptime_not_user_defined  = all_valid_comptime_idents()
+	array_builtin_methods            = ['filter', 'clone', 'repeat', 'reverse', 'map', 'slice',
 		'sort', 'contains', 'index', 'wait', 'any', 'all', 'first', 'last', 'pop']
-	reserved_type_names             = ['bool', 'char', 'i8', 'i16', 'int', 'i64', 'byte', 'u16',
+	reserved_type_names              = ['bool', 'char', 'i8', 'i16', 'int', 'i64', 'byte', 'u16',
 		'u32', 'u64', 'f32', 'f64', 'map', 'string', 'rune']
-	vroot_is_deprecated_message     = '@VROOT is deprecated, use @VMODROOT or @VEXEROOT instead'
+	vroot_is_deprecated_message      = '@VROOT is deprecated, use @VMODROOT or @VEXEROOT instead'
 )
 
 fn all_valid_comptime_idents() []string {
@@ -5866,7 +5867,12 @@ fn (mut c Checker) comptime_call(mut node ast.ComptimeCall) ast.Type {
 		return ast.string_type
 	}
 	if node.is_embed {
-		c.file.embedded_files << node.embed_file
+		// c.file.embedded_files << node.embed_file
+		if node.embed_file.compression_type !in checker.valid_comptime_compression_types {
+			supported := checker.valid_comptime_compression_types.map('.$it').join(', ')
+			c.error('not supported compression type: .${node.embed_file.compression_type}. supported: $supported',
+				node.pos)
+		}
 		return c.table.find_type_idx('v.embed_file.EmbedFileData')
 	}
 	if node.is_vweb {

vlib/v/checker/tests/embed_unknown_compress_type.out

@@ -0,0 +1,7 @@
+vlib/v/checker/tests/embed_unknown_compress_type.vv:3:10: error: not supported compression type: .this_is_not_a_valid_compression_type. supported: .none, .zlib
+    1 | 
+    2 | fn main() {
+    3 |     test := $embed_file('embed_unknown_compress_type.vv', .this_is_not_a_valid_compression_type)
+      |             ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    4 |     println(test.to_string())
+    5 | }

vlib/v/checker/tests/embed_unknown_compress_type.vv

@@ -0,0 +1,5 @@
+
+fn main() {
+	test := $embed_file('embed_unknown_compress_type.vv', .this_is_not_a_valid_compression_type)
+	println(test.to_string())
+}

vlib/v/embed_file/decoder.v

@@ -0,0 +1,17 @@
+[has_globals]
+module embed_file
+
+interface Decoder {
+	decompress([]byte) ?[]byte
+}
+
+struct EmbedFileDecoders {
+mut:
+	decoders map[string]Decoder
+}
+
+__global g_embed_file_decoders = &EmbedFileDecoders{}
+
+pub fn register_decoder(compression_type string, decoder Decoder) {
+	g_embed_file_decoders.decoders[compression_type] = decoder
+}

vlib/v/embed_file/embed_file.v

@@ -9,7 +9,8 @@ pub const (
 // https://github.com/vlang/rfcs/blob/master/embedding_resources.md
 // EmbedFileData encapsulates functionality for the `$embed_file()` compile time call.
 pub struct EmbedFileData {
-	apath string
+	apath            string
+	compression_type string
 mut:
 	compressed        &byte
 	uncompressed      &byte
@@ -29,6 +30,7 @@ pub fn (mut ed EmbedFileData) free() {
 	unsafe {
 		ed.path.free()
 		ed.apath.free()
+		ed.compression_type.free()
 		if ed.free_compressed {
 			free(ed.compressed)
 			ed.compressed = &byte(0)
@@ -59,25 +61,31 @@ pub fn (original &EmbedFileData) to_bytes() []byte {
 pub fn (mut ed EmbedFileData) data() &byte {
 	if !isnil(ed.uncompressed) {
 		return ed.uncompressed
+	}
+	if isnil(ed.uncompressed) && !isnil(ed.compressed) {
+		decoder := g_embed_file_decoders.decoders[ed.compression_type] or {
+			panic('EmbedFileData error: unknown compression of "$ed.path": "$ed.compression_type"')
+		}
+		compressed := unsafe { ed.compressed.vbytes(ed.len) }
+		decompressed := decoder.decompress(compressed) or {
+			panic('EmbedFileData error: decompression of "$ed.path" failed: $err')
+		}
+		unsafe {
+			ed.uncompressed = &byte(memdup(decompressed.data, ed.len))
+		}
 	} else {
-		if isnil(ed.uncompressed) && !isnil(ed.compressed) {
-			// TODO implement uncompression
-			// See also C Gen.gen_embedded_data() where the compression should occur.
-			ed.uncompressed = ed.compressed
-		} else {
-			mut path := os.resource_abs_path(ed.path)
+		mut path := os.resource_abs_path(ed.path)
+		if !os.is_file(path) {
+			path = ed.apath
 			if !os.is_file(path) {
-				path = ed.apath
-				if !os.is_file(path) {
-					panic('EmbedFileData error: files "$ed.path" and "$ed.apath" do not exist')
-				}
-			}
-			bytes := os.read_bytes(path) or {
-				panic('EmbedFileData error: "$path" could not be read: $err')
+				panic('EmbedFileData error: files "$ed.path" and "$ed.apath" do not exist')
 			}
-			ed.uncompressed = bytes.data
-			ed.free_uncompressed = true
 		}
+		bytes := os.read_bytes(path) or {
+			panic('EmbedFileData error: "$path" could not be read: $err')
+		}
+		ed.uncompressed = bytes.data
+		ed.free_uncompressed = true
 	}
 	return ed.uncompressed
 }
@@ -91,19 +99,20 @@ pub fn (mut ed EmbedFileData) data() &byte {
 pub struct EmbedFileIndexEntry {
 	id   int
 	path string
+	algo string
 	data &byte
 }
 
 // find_index_entry_by_path is used internally by the V compiler:
-pub fn find_index_entry_by_path(start voidptr, path string) &EmbedFileIndexEntry {
+pub fn find_index_entry_by_path(start voidptr, path string, algo string) &EmbedFileIndexEntry {
 	mut x := &EmbedFileIndexEntry(start)
-	for !(x.path == path || isnil(x.data)) {
+	for x.id >= 0 && x.data != 0 && (x.algo != algo || x.path != path) {
 		unsafe {
 			x++
 		}
 	}
 	$if debug_embed_file_in_prod ? {
-		eprintln('>> v.embed_file find_index_entry_by_path ${ptr_str(start)}, path: "$path" => ${ptr_str(x)}')
+		eprintln('>> v.embed_file find_index_entry_by_path ${ptr_str(start)}, id: $x.id, path: "$path", algo: "$algo" => ${ptr_str(x)}')
 	}
 	return x
 }

vlib/v/gen/c/cgen.v

@@ -3828,7 +3828,7 @@ fn (mut g Gen) expr(node ast.Expr) {
 		}
 		ast.Comment {}
 		ast.ComptimeCall {
-			g.comptime_call(node)
+			g.comptime_call(mut node)
 		}
 		ast.ComptimeSelector {
 			g.comptime_selector(node)