Add support for decompressor specific data.

Libraries can use this feature to store additional data needed by the kernel
decompressor. At this point it is only used by the xz decompressor to allow
users to set their own dictionary sizes.

README.md

@@ -73,7 +73,7 @@ has not been tested.
 
 `microfs` images has four main parts:
 
- * `super block`
+ * `super block/decompressor data`
  * `inodes/dentries`
  * `block pointers`
  * `compressed data blocks`
@@ -86,6 +86,12 @@ its buffers more efficiently. The downside is that each regular
 file and symlink requires one extra block pointer with this
 setup.
 
+It is possible that decompressor specific data is stored
+immediately after the super block. This will happen if the
+decompressor needs some extra information that is not
+normally available, one example is the dictionary size for
+the `xz` decompressor.
+
 ## Building microfs
 
 It is presumed that you are planning to build `microfs` on a

hostprog_microfscki.c

@@ -245,20 +245,6 @@ static void ck_sb(struct imgdesc* const desc)
 		desc->de_quickie = 1;
 	}
 
-	const __u64 expected_root_offset = padding + sizeof(*desc->de_sb);
-	const __u64 actual_root_offset = __le32_to_cpu(desc->de_sb->s_root.i_offset);
-
-	const int invalid_offset = (
-		actual_root_offset != 0 &&
-		actual_root_offset != expected_root_offset
-	);
-
-	if (invalid_offset) {
-		error("invalid root offset value");
-	}
-
-	desc->de_metadatasz = expected_root_offset;
-
 	desc->de_blkshift = __le16_to_cpu(desc->de_sb->s_blkshift);
 	desc->de_blksz = 1 << desc->de_blkshift;
 
@@ -282,6 +268,25 @@ static void ck_sb(struct imgdesc* const desc)
 		desc->de_decompressionbuf = malloc(desc->de_decompressionbufsz);
 		if (!desc->de_decompressionbuf)
 			error("failed to allocate the decompression buffer");
+
+		const __u64 actual_root_offset = __le32_to_cpu(desc->de_sb->s_root.i_offset);
+		const __u64 expected_root_offset = padding + sizeof(*desc->de_sb)
+			+ desc->de_lib->hl_info->li_dd_sz;
+
+		const int invalid_offset = (
+			actual_root_offset != 0 &&
+			actual_root_offset != expected_root_offset
+		);
+
+		if (invalid_offset) {
+			error("invalid root offset value");
+		}
+
+		desc->de_metadatasz = expected_root_offset;
+
+		__u64 dd_offset = padding + sizeof(*desc->de_sb);
+		if (desc->de_lib->hl_ck_dd(desc->de_lib_data, desc->de_image + dd_offset) < 0)
+			error("decompressor specific data check failed");
 	}
 }
 

hostprog_microfsmki.c

@@ -371,7 +371,9 @@ static void write_superblock(struct imgspec* const spec, char* base,
 	const __u64 sz)
 {
 	__u64 padding = superblock_offset(spec);
-	__u64 offset = padding + sizeof(struct microfs_sb);
+	__u64 offset = padding + sizeof(struct microfs_sb)
+		+ spec->sp_lib->hl_info->li_dd_sz;
+
 	struct microfs_sb* sb = (struct microfs_sb*)(base + padding);
 
 	sb->s_magic = __cpu_to_le32(MICROFS_MAGIC);
@@ -415,6 +417,14 @@ static void write_superblock(struct imgspec* const spec, char* base,
 	message(VERBOSITY_0, "CRC: %x", crc);
 }
 
+/* Write any decompressor specific data.
+ */
+static __u64 write_decompressordata(struct imgspec* const spec,
+	char* base, __u64 offset)
+{
+	return spec->sp_lib->hl_mk_dd(spec->sp_lib_data, base, offset);
+}
+
 /* Write metadata for the given entries, but not their actual
  * data (see write_data() for that).
  */
@@ -1004,6 +1014,8 @@ static struct imgspec* create_imgspec(int argc, char* argv[])
 
 	lib_options(spec);
 
+	spec->sp_upperbound += spec->sp_lib->hl_info->li_dd_sz;
+
 	if (spec->sp_lib->hl_info->li_min_blksz == 0 && spec->sp_blksz < spec->sp_pagesz) {
 		warning("block size smaller than page size of host"
 			" - the resulting image can not be used on this host");
@@ -1108,6 +1120,7 @@ static void materialize_imgspec(struct imgspec* const spec)
 
 	__u64 offset = superblock_offset(spec) + sizeof(struct microfs_sb);
 
+	offset = write_decompressordata(spec, image, offset);
 	offset = write_metadata(spec, image, offset);
 	offset = write_data(spec, image, offset);
 

hostprogs_lib.c

@@ -65,10 +65,9 @@ const struct hostprog_lib** hostprog_lib_all(void)
 	return hostprog_libs;
 }
 
-int hostprog_lib_mk_usage(FILE* const dest)
+void hostprog_lib_mk_usage(FILE* const dest)
 {
 	fprintf(dest, " no options available\n");
-	return 0;
 }
 
 int hostprog_lib_mk_option(void* data, const char* name, const char* value)
@@ -79,3 +78,17 @@ int hostprog_lib_mk_option(void* data, const char* name, const char* value)
 	return 0;
 }
 
+__u64 hostprog_lib_mk_dd(void* data, char* base, __u64 offset)
+{
+	(void)data;
+	(void)base;
+	return offset;
+}
+
+int hostprog_lib_ck_dd(void* data, char* dd)
+{
+	(void)data;
+	(void)dd;
+	return 0;
+}
+

hostprogs_lib.h

@@ -21,6 +21,7 @@
 
 #include <stdio.h>
 
+#include <asm/byteorder.h>
 #include <linux/types.h>
 
 #include "libinfo.h"
@@ -33,9 +34,13 @@ struct hostprog_lib {
 	/* Compression library initialization, if necessary. */
 	int (*hl_init)(void** data, __u32 blksz);
 	/* Compression library help. */
-	int (*hl_mk_usage)(FILE* const dest);
+	void (*hl_mk_usage)(FILE* const dest);
 	/* Compression library option. */
 	int (*hl_mk_option)(void* data, const char* name, const char* value);
+	/* Write "on-disk" decompressor data. */
+	__u64 (*hl_mk_dd)(void* data, char* base, __u64 offset);
+	/* Check "on-disk" decompressor data. */
+	int (*hl_ck_dd)(void* data, char* dd);
 	/* Compress data. */
 	int (*hl_compress)(void* data, void* destbuf, __u32* destbufsz,
 		void* srcbuf, __u32 srcbufsz, int* implerr);
@@ -58,8 +63,10 @@ const struct hostprog_lib* hostprog_lib_find_byname(const char* name);
 
 const struct hostprog_lib** hostprog_lib_all(void);
 
-int hostprog_lib_mk_usage(FILE* const dest);
+void hostprog_lib_mk_usage(FILE* const dest);
 int hostprog_lib_mk_option(void* data, const char* name, const char* value);
+__u64 hostprog_lib_mk_dd(void* data, char* base, __u64 offset);
+int hostprog_lib_ck_dd(void* data, char* base);
 
 __u32 hostprog_lib_zlib_crc32(char* data, __u64 sz);
 

hostprogs_lib_lz4.c

@@ -55,12 +55,11 @@ static int hostprog_lib_lz4_init(void** data, __u32 blksz)
 }
 
 
-static int hostprog_lib_lz4_mk_usage(FILE* const dest)
+static void hostprog_lib_lz4_mk_usage(FILE* const dest)
 {
 	fprintf(dest,
 		" compression=<str>     select compression level (default, high)\n"
 	);
-	return 0;
 }
 
 static int hostprog_lib_lz4_mk_option(void* data,
@@ -89,9 +88,9 @@ static int hostprog_lib_lz4_mk_option(void* data,
 static int hostprog_lib_lz4_compress(void* data, void* destbuf, __u32* destbufsz,
 	void* srcbuf, __u32 srcbufsz, int* implerr)
 {
-	struct hostprog_lib_lz4_data* opts = data;
+	struct hostprog_lib_lz4_data* lz4_data = data;
 
-	*implerr = opts->d_compressor(srcbuf, destbuf, srcbufsz, *destbufsz);
+	*implerr = lz4_data->d_compressor(srcbuf, destbuf, srcbufsz, *destbufsz);
 	*destbufsz = *implerr? *implerr: 0;
 	return *implerr != 0? 0: -1;
 }
@@ -127,6 +126,8 @@ const struct hostprog_lib hostprog_lib_lz4 = {
 	.hl_init = hostprog_lib_lz4_init,
 	.hl_mk_usage = hostprog_lib_lz4_mk_usage,
 	.hl_mk_option = hostprog_lib_lz4_mk_option,
+	.hl_mk_dd = hostprog_lib_mk_dd,
+	.hl_ck_dd = hostprog_lib_ck_dd,
 	.hl_compress = hostprog_lib_lz4_compress,
 	.hl_decompress = hostprog_lib_lz4_decompress,
 	.hl_upperbound = hostprog_lib_lz4_upperbound,

hostprogs_lib_lzo.c

@@ -85,6 +85,8 @@ const struct hostprog_lib hostprog_lib_lzo = {
 	.hl_init = hostprog_lib_lzo_init,
 	.hl_mk_usage = hostprog_lib_mk_usage,
 	.hl_mk_option = hostprog_lib_mk_option,
+	.hl_mk_dd = hostprog_lib_mk_dd,
+	.hl_ck_dd = hostprog_lib_ck_dd,
 	.hl_compress = hostprog_lib_lzo_compress,
 	.hl_decompress = hostprog_lib_lzo_decompress,
 	.hl_upperbound = hostprog_lib_lzo_upperbound,

hostprogs_lib_xz.c

@@ -91,20 +91,37 @@ static int hostprog_lib_xz_init(void** data, __u32 blksz)
 	return 0;
 }
 
-static int hostprog_lib_xz_mk_usage(FILE* const dest)
+static void hostprog_lib_xz_mk_usage(FILE* const dest)
 {
 	fprintf(dest,
+		" dictionary=<int>      dictionary size (power of two, bigger than %u)\n"
 		" filter=<str>          add a BCJ conversion filter"
 		" (x86, powerpc, ia64, arm, armthumb, sparc)\n"
+		"", LZMA_DICT_SIZE_MIN
 	);
-	return 0;
 }
 
 static int hostprog_lib_xz_mk_option(void* data,
 	const char* name, const char* value)
 {
 	struct hostprog_lib_xz_data* xz_data = data;
 
+	if (strcmp(name, "dictionary") == 0) {
+		if (!value) {
+			errno = EINVAL;
+			goto err;
+		}
+
+		char* endptr;
+		xz_data->d_opts.dict_size = strtoul(value, &endptr, 10);
+		if (*endptr != '\0') {
+			errno = EINVAL;
+			goto err;
+		}
+
+		return 0;
+	}
+
 	if (strcmp(name, "filter") == 0) {
 		if (!value) {
 			errno = EINVAL;
@@ -148,6 +165,7 @@ static int hostprog_lib_xz_mk_option(void* data,
 			errno = ENOMEM;
 			goto err;
 		}
+
 		return 0;
 	}
 
@@ -158,6 +176,32 @@ static int hostprog_lib_xz_mk_option(void* data,
 	return -1;
 }
 
+static __u64 hostprog_lib_xz_mk_dd(void* data, char* base, __u64 offset)
+{
+	struct microfs_dd_xz* dd_xz = (struct microfs_dd_xz*)(base + offset);
+	struct hostprog_lib_xz_data* xz_data = data;
+
+	dd_xz->dd_magic = __cpu_to_le32(MICROFS_DD_XZ_MAGIC);
+	dd_xz->dd_dictsz = __cpu_to_le32(xz_data->d_opts.dict_size);
+
+	return offset + hostprog_lib_xz.hl_info->li_dd_sz;
+}
+
+static int hostprog_lib_xz_ck_dd(void* data, char* dd)
+{
+	int err = -1;
+	struct microfs_dd_xz* dd_xz = (struct microfs_dd_xz*)dd;
+	struct hostprog_lib_xz_data* xz_data = data;
+
+	if (__le32_to_cpu(dd_xz->dd_magic) == MICROFS_DD_XZ_MAGIC) {
+		xz_data->d_opts.dict_size = __le32_to_cpu(dd_xz->dd_dictsz);
+		if (xz_data->d_opts.dict_size >= LZMA_DICT_SIZE_MIN)
+			err = 0;
+	}
+
+	return err;
+}
+
 static int hostprog_lib_xz_compress(void* data, void* destbuf, __u32* destbufsz,
 	void* srcbuf, __u32 srcbufsz, int* implerr)
 {
@@ -249,6 +293,8 @@ const struct hostprog_lib hostprog_lib_xz = {
 	.hl_init = hostprog_lib_xz_init,
 	.hl_mk_usage = hostprog_lib_xz_mk_usage,
 	.hl_mk_option = hostprog_lib_xz_mk_option,
+	.hl_mk_dd = hostprog_lib_xz_mk_dd,
+	.hl_ck_dd = hostprog_lib_xz_ck_dd,
 	.hl_compress = hostprog_lib_xz_compress,
 	.hl_decompress = hostprog_lib_xz_decompress,
 	.hl_upperbound = hostprog_lib_xz_upperbound,

hostprogs_lib_zlib.c

@@ -49,12 +49,11 @@ static int hostprog_lib_zlib_init(void** data, __u32 blksz)
 	return 0;
 }
 
-static int hostprog_lib_zlib_mk_usage(FILE* const dest)
+static void hostprog_lib_zlib_mk_usage(FILE* const dest)
 {
 	fprintf(dest,
 		" compression=<str>     select compression level (default, size, speed, none)\n"
 	);
-	return 0;
 }
 
 static int hostprog_lib_zlib_mk_option(void* data,
@@ -121,6 +120,8 @@ const struct hostprog_lib hostprog_lib_zlib = {
 	.hl_init = hostprog_lib_zlib_init,
 	.hl_mk_usage = hostprog_lib_zlib_mk_usage,
 	.hl_mk_option = hostprog_lib_zlib_mk_option,
+	.hl_mk_dd = hostprog_lib_mk_dd,
+	.hl_ck_dd = hostprog_lib_ck_dd,
 	.hl_compress = hostprog_lib_zlib_compress,
 	.hl_decompress = hostprog_lib_zlib_decompress,
 	.hl_upperbound = hostprog_lib_zlib_upperbound,

libinfo.h

@@ -28,6 +28,7 @@ struct libinfo {
 	const int li_streaming;
 	const __u32 li_min_blksz;
 	const __u32 li_max_blksz;
+	const __u32 li_dd_sz;
 	const char* li_name;
 };
 

libinfo_lz4.h

@@ -26,6 +26,7 @@ static const struct libinfo libinfo_lz4 = {
 	.li_streaming = 0,
 	.li_min_blksz = 0,
 	.li_max_blksz = MICROFS_MAXBLKSZ,
+	.li_dd_sz = 0,
 	.li_name = "lz4"
 };
 

libinfo_lzo.h

@@ -26,6 +26,7 @@ static const struct libinfo libinfo_lzo = {
 	.li_streaming = 0,
 	.li_min_blksz = 0,
 	.li_max_blksz = MICROFS_MAXBLKSZ,
+	.li_dd_sz = 0,
 	.li_name = "lzo"
 };
 

libinfo_xz.h

@@ -23,11 +23,17 @@
 
 #define LIBINFO_XZ_DICTSZ MICROFS_MAXBLKSZ
 
+struct microfs_dd_xz {
+	__le32 dd_magic;
+	__le32 dd_dictsz;
+}  __attribute__ ((packed));
+
 static const struct libinfo libinfo_xz = {
 	.li_id = MICROFS_FLAG_DECOMPRESSOR_XZ,
 	.li_streaming = 1,
 	.li_min_blksz = MICROFS_MINBLKSZ,
 	.li_max_blksz = MICROFS_MAXBLKSZ,
+	.li_dd_sz = sizeof(struct microfs_dd_xz),
 	.li_name = "xz"
 };
 

libinfo_zlib.h

@@ -26,6 +26,7 @@ static const struct libinfo libinfo_zlib = {
 	.li_streaming = 1,
 	.li_min_blksz = MICROFS_MINBLKSZ,
 	.li_max_blksz = MICROFS_MAXBLKSZ,
+	.li_dd_sz = 0,
 	.li_name = "zlib"
 };