| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| COMMON_H = \ | ||
| $(top_srcdir)/include/sys/zstd/zstd.h | ||
|
|
||
| KERNEL_H = | ||
|
|
||
| USER_H = | ||
|
|
||
| EXTRA_DIST = $(COMMON_H) $(KERNEL_H) $(USER_H) | ||
|
|
||
| if CONFIG_USER | ||
| libzfsdir = $(includedir)/libzfs/sys/zstd | ||
| libzfs_HEADERS = $(COMMON_H) $(USER_H) | ||
| endif | ||
|
|
||
| if CONFIG_KERNEL | ||
| kerneldir = @prefix@/src/zfs-$(VERSION)/include/sys/zstd | ||
| kernel_HEADERS = $(COMMON_H) $(KERNEL_H) | ||
| endif |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| /* | ||
| * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) | ||
| * | ||
| * Redistribution and use in source and binary forms, with or without | ||
| * modification, are permitted provided that the following conditions are met: | ||
| * | ||
| * 1. Redistributions of source code must retain the above copyright notice, | ||
| * this list of conditions and the following disclaimer. | ||
| * | ||
| * 2. Redistributions in binary form must reproduce the above copyright notice, | ||
| * this list of conditions and the following disclaimer in the documentation | ||
| * and/or other materials provided with the distribution. | ||
| * | ||
| * 3. Neither the name of the copyright holder nor the names of its | ||
| * contributors may be used to endorse or promote products derived from this | ||
| * software without specific prior written permission. | ||
| * | ||
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE | ||
| * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
| * POSSIBILITY OF SUCH DAMAGE. | ||
| */ | ||
|
|
||
| /* | ||
| * Copyright (c) 2016-2018, Klara Inc. | ||
| * Copyright (c) 2016-2018, Allan Jude | ||
| * Copyright (c) 2018-2020, Sebastian Gottschall | ||
| * Copyright (c) 2019-2020, Michael Niewöhner | ||
| * Copyright (c) 2020, The FreeBSD Foundation [1] | ||
| * | ||
| * [1] Portions of this software were developed by Allan Jude | ||
| * under sponsorship from the FreeBSD Foundation. | ||
| */ | ||
|
|
||
| #ifndef _ZFS_ZSTD_H | ||
| #define _ZFS_ZSTD_H | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| /* | ||
| * ZSTD block header | ||
| * NOTE: all fields in this header are in big endian order. | ||
| */ | ||
| typedef struct zfs_zstd_header { | ||
| /* Compressed size of data */ | ||
| uint32_t c_len; | ||
|
|
||
| /* | ||
| * Version and compression level | ||
| * We use a union to be able to big endian encode a single 32 bit | ||
| * unsigned integer, but still access the individual bitmasked | ||
| * components easily. | ||
| */ | ||
| union { | ||
| uint32_t raw_version_level; | ||
| struct { | ||
| uint32_t version : 24; | ||
| uint8_t level; | ||
| }; | ||
| }; | ||
|
|
||
| char data[]; | ||
| } zfs_zstdhdr_t; | ||
|
|
||
| /* | ||
| * kstat helper macros | ||
| */ | ||
| #define ZSTDSTAT(stat) (zstd_stats.stat.value.ui64) | ||
| #define ZSTDSTAT_INCR(stat, val) \ | ||
| atomic_add_64(&zstd_stats.stat.value.ui64, (val)) | ||
| #define ZSTDSTAT_BUMP(stat) ZSTDSTAT_INCR(stat, 1) | ||
|
|
||
| /* (de)init for user space / kernel emulation */ | ||
| int zstd_init(void); | ||
| void zstd_fini(void); | ||
|
|
||
| size_t zstd_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, | ||
| int level); | ||
| int zstd_get_level(void *s_start, size_t s_len, uint8_t *level); | ||
| int zstd_decompress_level(void *s_start, void *d_start, size_t s_len, | ||
| size_t d_len, uint8_t *level); | ||
| int zstd_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, | ||
| int n); | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif /* _ZFS_ZSTD_H */ |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| include $(top_srcdir)/config/Rules.am | ||
|
|
||
| VPATH = $(top_srcdir)/module/zstd | ||
|
|
||
| # Includes kernel code, generate warnings for large stack frames | ||
| AM_CFLAGS += $(FRAME_LARGER_THAN) | ||
|
|
||
| noinst_LTLIBRARIES = libzstd.la | ||
|
|
||
| KERNEL_C = \ | ||
| lib/zstd.c \ | ||
| zfs_zstd.c | ||
|
|
||
| nodist_libzstd_la_SOURCES = $(KERNEL_C) | ||
|
|
||
| # -fno-tree-vectorize is set for gcc in zstd/common/compiler.h | ||
| # Set it for other compilers, too. | ||
| lib/zstd.$(OBJEXT): CFLAGS += -fno-tree-vectorize | ||
| lib/zstd.l$(OBJEXT): CFLAGS += -fno-tree-vectorize | ||
|
|
||
| # Quiet warnings about frame size due to unused code in unmodified zstd lib | ||
| lib/zstd.$(OBJEXT): CFLAGS += -Wframe-larger-than=20480 | ||
| lib/zstd.l$(OBJEXT): CFLAGS += -Wframe-larger-than=20480 |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| ifneq ($(KBUILD_EXTMOD),) | ||
| src = @abs_srcdir@ | ||
| obj = @abs_builddir@ | ||
| zstd_include = $(src)/include | ||
| else | ||
| zstd_include = $(srctree)/$(src)/include | ||
| endif | ||
|
|
||
| MODULE := zzstd | ||
|
|
||
| obj-$(CONFIG_ZFS) := $(MODULE).o | ||
|
|
||
| asflags-y := -I$(zstd_include) | ||
| ccflags-y := -I$(zstd_include) | ||
|
|
||
| # Zstd uses -O3 by default, so we should follow | ||
| ccflags-y += -O3 | ||
|
|
||
| # -fno-tree-vectorize gets set for gcc in zstd/common/compiler.h | ||
| # Set it for other compilers, too. | ||
| $(obj)/lib/zstd.o: c_flags += -fno-tree-vectorize | ||
|
|
||
| # Quiet warnings about frame size due to unused code in unmodified zstd lib | ||
| $(obj)/lib/zstd.o: c_flags += -Wframe-larger-than=20480 | ||
|
|
||
| # Disable aarch64 neon SIMD instructions for kernel mode | ||
| $(obj)/lib/zstd.o: c_flags += -include $(zstd_include)/aarch64_compat.h | ||
|
|
||
| $(MODULE)-objs += zfs_zstd.o | ||
| $(MODULE)-objs += lib/zstd.o | ||
|
|
||
| all: | ||
| mkdir -p lib |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| # ZSTD-On-ZFS Library Manual | ||
|
|
||
| ## Introduction | ||
|
|
||
| This subtree contains the ZSTD library used in ZFS. It is heavily cut-down by | ||
| dropping any unneeded files, and combined into a single file, but otherwise is | ||
| intentionally unmodified. Please do not alter the file containing the zstd | ||
| library, besides upgrading to a newer ZSTD release. | ||
|
|
||
| Tree structure: | ||
|
|
||
| * `zfs_zstd.c` is the actual `zzstd` kernel module. | ||
| * `lib/` contains the the unmodified, [_"amalgamated"_](https://github.com/facebook/zstd/blob/dev/contrib/single_file_libs/README.md) | ||
| version of the `Zstandard` library, generated from our template file | ||
| * `zstd-in.c` is our template file for generating the library | ||
| * `include/`: This directory contains supplemental includes for platform | ||
| compatibility, which are not expected to be used by ZFS elsewhere in the | ||
| future. Thus we keep them private to ZSTD. | ||
|
|
||
| ## Updating ZSTD | ||
|
|
||
| To update ZSTD the following steps need to be taken: | ||
|
|
||
| 1. Grab the latest release of [ZSTD](https://github.com/facebook/zstd/releases). | ||
| 2. Update `module/zstd/zstd-in.c` if required. (see | ||
| `zstd/contrib/single_file_libs/zstd-in.c` in the zstd repository) | ||
| 3. Generate the "single-file-library" and put it to `module/zstd/lib/`. | ||
| 4. Copy the following files to `module/zstd/lib/`: | ||
| - `zstd/lib/zstd.h` | ||
| - `zstd/lib/common/zstd_errors.h` | ||
|
|
||
| This can be done using a few shell commands from inside the zfs repo: | ||
|
|
||
| ~~~sh | ||
| cd PATH/TO/ZFS | ||
|
|
||
| url="https://github.com/facebook/zstd" | ||
| release="$(curl -s "${url}"/releases/latest | grep -oP '(?<=v)[\d\.]+')" | ||
| zstd="/tmp/zstd-${release}/" | ||
|
|
||
| wget -O /tmp/zstd.tar.gz \ | ||
| "${url}/releases/download/v${release}/zstd-${release}.tar.gz" | ||
| tar -C /tmp -xzf /tmp/zstd.tar.gz | ||
|
|
||
| cp ${zstd}/lib/zstd.h module/zstd/lib/ | ||
| cp ${zstd}/lib/zstd_errors.h module/zstd/lib/ | ||
| ${zstd}/contrib/single_file_libs/combine.sh \ | ||
| -r ${zstd}/lib -o module/zstd/lib/zstd.c module/zstd/zstd-in.c | ||
| ~~~ | ||
|
|
||
|
|
||
| ## Altering ZSTD and breaking changes | ||
|
|
||
| If ZSTD made changes that break compatibility or you need to make breaking | ||
| changes to the way we handle ZSTD, it is required to maintain backwards | ||
| compatibility. | ||
|
|
||
| We already save the ZSTD version number within the block header to be used | ||
| to add future compatibility checks and/or fixes. However, currently it is | ||
| not actually used in such a way. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| /* | ||
| * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) | ||
| * | ||
| * Redistribution and use in source and binary forms, with or without | ||
| * modification, are permitted provided that the following conditions are met: | ||
| * | ||
| * 1. Redistributions of source code must retain the above copyright notice, | ||
| * this list of conditions and the following disclaimer. | ||
| * | ||
| * 2. Redistributions in binary form must reproduce the above copyright notice, | ||
| * this list of conditions and the following disclaimer in the documentation | ||
| * and/or other materials provided with the distribution. | ||
| * | ||
| * 3. Neither the name of the copyright holder nor the names of its | ||
| * contributors may be used to endorse or promote products derived from this | ||
| * software without specific prior written permission. | ||
| * | ||
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE | ||
| * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
| * POSSIBILITY OF SUCH DAMAGE. | ||
| */ | ||
|
|
||
| /* | ||
| * Copyright (c) 2018-2020, Sebastian Gottschall | ||
| */ | ||
|
|
||
| #ifdef _KERNEL | ||
| #undef __aarch64__ | ||
| #endif |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| /* | ||
| * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) | ||
| * | ||
| * Redistribution and use in source and binary forms, with or without | ||
| * modification, are permitted provided that the following conditions are met: | ||
| * | ||
| * 1. Redistributions of source code must retain the above copyright notice, | ||
| * this list of conditions and the following disclaimer. | ||
| * | ||
| * 2. Redistributions in binary form must reproduce the above copyright notice, | ||
| * this list of conditions and the following disclaimer in the documentation | ||
| * and/or other materials provided with the distribution. | ||
| * | ||
| * 3. Neither the name of the copyright holder nor the names of its | ||
| * contributors may be used to endorse or promote products derived from this | ||
| * software without specific prior written permission. | ||
| * | ||
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE | ||
| * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
| * POSSIBILITY OF SUCH DAMAGE. | ||
| */ | ||
|
|
||
| /* | ||
| * Copyright (c) 2014-2019, Allan Jude | ||
| * Copyright (c) 2020, Brian Behlendorf | ||
| * Copyright (c) 2020, Michael Niewöhner | ||
| */ | ||
|
|
||
| #ifndef _ZSTD_LIMITS_H | ||
| #define _ZSTD_LIMITS_H | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| #ifdef _KERNEL | ||
|
|
||
| #if defined(__FreeBSD__) | ||
| #include <sys/limits.h> | ||
| #elif defined(__linux__) | ||
| #include <linux/limits.h> | ||
| #include <linux/kernel.h> | ||
| #else | ||
| #error "Unsupported platform" | ||
| #endif | ||
|
|
||
| #else /* !_KERNEL */ | ||
| #include_next <limits.h> | ||
| #endif /* _KERNEL */ | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif /* _ZSTD_LIMITS_H */ |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| /* | ||
| * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) | ||
| * | ||
| * Redistribution and use in source and binary forms, with or without | ||
| * modification, are permitted provided that the following conditions are met: | ||
| * | ||
| * 1. Redistributions of source code must retain the above copyright notice, | ||
| * this list of conditions and the following disclaimer. | ||
| * | ||
| * 2. Redistributions in binary form must reproduce the above copyright notice, | ||
| * this list of conditions and the following disclaimer in the documentation | ||
| * and/or other materials provided with the distribution. | ||
| * | ||
| * 3. Neither the name of the copyright holder nor the names of its | ||
| * contributors may be used to endorse or promote products derived from this | ||
| * software without specific prior written permission. | ||
| * | ||
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE | ||
| * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
| * POSSIBILITY OF SUCH DAMAGE. | ||
| */ | ||
|
|
||
| /* | ||
| * Copyright (c) 2014-2019, Allan Jude | ||
| * Copyright (c) 2020, Brian Behlendorf | ||
| * Copyright (c) 2020, Michael Niewöhner | ||
| */ | ||
|
|
||
| #ifndef _ZSTD_STDDEF_H | ||
| #define _ZSTD_STDDEF_H | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| #ifdef _KERNEL | ||
|
|
||
| #if defined(__FreeBSD__) | ||
| #include <sys/types.h> | ||
| #elif defined(__linux__) | ||
| #include <linux/types.h> | ||
| #else | ||
| #error "Unsupported platform" | ||
| #endif | ||
|
|
||
| #else /* !_KERNEL */ | ||
| #include_next <stddef.h> | ||
| #endif /* _KERNEL */ | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif /* _ZSTD_STDDEF_H */ |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| /* | ||
| * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) | ||
| * | ||
| * Redistribution and use in source and binary forms, with or without | ||
| * modification, are permitted provided that the following conditions are met: | ||
| * | ||
| * 1. Redistributions of source code must retain the above copyright notice, | ||
| * this list of conditions and the following disclaimer. | ||
| * | ||
| * 2. Redistributions in binary form must reproduce the above copyright notice, | ||
| * this list of conditions and the following disclaimer in the documentation | ||
| * and/or other materials provided with the distribution. | ||
| * | ||
| * 3. Neither the name of the copyright holder nor the names of its | ||
| * contributors may be used to endorse or promote products derived from this | ||
| * software without specific prior written permission. | ||
| * | ||
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE | ||
| * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
| * POSSIBILITY OF SUCH DAMAGE. | ||
| */ | ||
|
|
||
| /* | ||
| * Copyright (c) 2014-2019, Allan Jude | ||
| * Copyright (c) 2020, Brian Behlendorf | ||
| * Copyright (c) 2020, Michael Niewöhner | ||
| */ | ||
|
|
||
| #ifndef _ZSTD_STDINT_H | ||
| #define _ZSTD_STDINT_H | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| #ifdef _KERNEL | ||
|
|
||
| #if defined(__FreeBSD__) | ||
| #include <sys/stdint.h> | ||
| #elif defined(__linux__) | ||
| #include <linux/types.h> | ||
| #else | ||
| #error "Unsupported platform" | ||
| #endif | ||
|
|
||
| #else /* !_KERNEL */ | ||
| #include_next <stdint.h> | ||
| #endif /* _KERNEL */ | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif /* _ZSTD_STDINT_H */ |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| /* | ||
| * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) | ||
| * | ||
| * Redistribution and use in source and binary forms, with or without | ||
| * modification, are permitted provided that the following conditions are met: | ||
| * | ||
| * 1. Redistributions of source code must retain the above copyright notice, | ||
| * this list of conditions and the following disclaimer. | ||
| * | ||
| * 2. Redistributions in binary form must reproduce the above copyright notice, | ||
| * this list of conditions and the following disclaimer in the documentation | ||
| * and/or other materials provided with the distribution. | ||
| * | ||
| * 3. Neither the name of the copyright holder nor the names of its | ||
| * contributors may be used to endorse or promote products derived from this | ||
| * software without specific prior written permission. | ||
| * | ||
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE | ||
| * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
| * POSSIBILITY OF SUCH DAMAGE. | ||
| */ | ||
|
|
||
| /* | ||
| * Copyright (c) 2014-2019, Allan Jude | ||
| * Copyright (c) 2020, Brian Behlendorf | ||
| * Copyright (c) 2020, Michael Niewöhner | ||
| */ | ||
|
|
||
| #ifndef _ZSTD_STDIO_H | ||
| #define _ZSTD_STDIO_H | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| #ifndef _KERNEL | ||
|
|
||
| #include_next <stdio.h> | ||
|
|
||
| #endif /* _KERNEL */ | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif /* _ZSTD_STDIO_H */ |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| /* | ||
| * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) | ||
| * | ||
| * Redistribution and use in source and binary forms, with or without | ||
| * modification, are permitted provided that the following conditions are met: | ||
| * | ||
| * 1. Redistributions of source code must retain the above copyright notice, | ||
| * this list of conditions and the following disclaimer. | ||
| * | ||
| * 2. Redistributions in binary form must reproduce the above copyright notice, | ||
| * this list of conditions and the following disclaimer in the documentation | ||
| * and/or other materials provided with the distribution. | ||
| * | ||
| * 3. Neither the name of the copyright holder nor the names of its | ||
| * contributors may be used to endorse or promote products derived from this | ||
| * software without specific prior written permission. | ||
| * | ||
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE | ||
| * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
| * POSSIBILITY OF SUCH DAMAGE. | ||
| */ | ||
|
|
||
| /* | ||
| * Copyright (c) 2014-2019, Allan Jude | ||
| * Copyright (c) 2020, Brian Behlendorf | ||
| * Copyright (c) 2020, Michael Niewöhner | ||
| */ | ||
|
|
||
| #ifndef _ZSTD_STDLIB_H | ||
| #define _ZSTD_STDLIB_H | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| #undef GCC_VERSION | ||
|
|
||
| /* | ||
| * Define calloc, malloc, free to make building work. They are never really used | ||
| * in zstdlib.c since allocation is done in zstd.c. | ||
| */ | ||
| #define calloc(n, sz) NULL | ||
| #define malloc(sz) NULL | ||
| #define free(ptr) | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif /* _ZSTD_STDLIB_H */ |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| /* | ||
| * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) | ||
| * | ||
| * Redistribution and use in source and binary forms, with or without | ||
| * modification, are permitted provided that the following conditions are met: | ||
| * | ||
| * 1. Redistributions of source code must retain the above copyright notice, | ||
| * this list of conditions and the following disclaimer. | ||
| * | ||
| * 2. Redistributions in binary form must reproduce the above copyright notice, | ||
| * this list of conditions and the following disclaimer in the documentation | ||
| * and/or other materials provided with the distribution. | ||
| * | ||
| * 3. Neither the name of the copyright holder nor the names of its | ||
| * contributors may be used to endorse or promote products derived from this | ||
| * software without specific prior written permission. | ||
| * | ||
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE | ||
| * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
| * POSSIBILITY OF SUCH DAMAGE. | ||
| */ | ||
|
|
||
| /* | ||
| * Copyright (c) 2014-2019, Allan Jude | ||
| * Copyright (c) 2020, Brian Behlendorf | ||
| * Copyright (c) 2020, Michael Niewöhner | ||
| */ | ||
|
|
||
| #ifndef _ZSTD_STRING_H | ||
| #define _ZSTD_STRING_H | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| #ifdef _KERNEL | ||
|
|
||
| #if defined(__FreeBSD__) | ||
| #include <sys/systm.h> /* memcpy, memset */ | ||
| #elif defined(__linux__) | ||
| #include <linux/string.h> /* memcpy, memset */ | ||
| #else | ||
| #error "Unsupported platform" | ||
| #endif | ||
|
|
||
| #else /* !_KERNEL */ | ||
| #include_next <string.h> | ||
| #endif /* _KERNEL */ | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif /* _ZSTD_STRING_H */ |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| #!/bin/ksh | ||
|
|
||
| # | ||
| # This file and its contents are supplied under the terms of the | ||
| # Common Development and Distribution License ("CDDL"), version 1.0. | ||
| # You may only use this file in accordance with the terms of version | ||
| # 1.0 of the CDDL. | ||
| # | ||
| # A full copy of the text of the CDDL should have accompanied this | ||
| # source. A copy of the CDDL is also available via the Internet at | ||
| # http://www.illumos.org/license/CDDL. | ||
| # | ||
|
|
||
| # | ||
| # Copyright (c) 2020 The FreeBSD Foundation [1] | ||
| # | ||
| # [1] Portions of this software were developed by Allan Jude | ||
| # under sponsorship from the FreeBSD Foundation. | ||
|
|
||
| . $STF_SUITE/include/libtest.shlib | ||
|
|
||
| # | ||
| # Description: | ||
| # zdb -Z pool <objid> will display the ZSTD compression header | ||
| # This will contain the actual length of the compressed data, as well as | ||
| # the version of ZSTD used to compress the block, and the compression level | ||
| # | ||
| # Strategy: | ||
| # 1. Create a pool, set compression to zstd-<random level> | ||
| # 2. Write some identifiable data to a file | ||
| # 3. Run zdb -Zddddddbbbbbb against the file | ||
| # 4. Record the DVA, lsize, and psize, and ZSTD header of L0 block 0 | ||
| # 5. Check that the ZSTD length is less than psize | ||
| # 6. Check that the ZSTD level matches the level we requested | ||
| # 7. Run zdb -R with :dr flags and confirm the size and content match | ||
| # | ||
|
|
||
| function cleanup | ||
| { | ||
| datasetexists $TESTPOOL && destroy_pool $TESTPOOL | ||
| } | ||
|
|
||
| log_assert "Verify zdb -Z (read ZSTD header) works as expected" | ||
| log_onexit cleanup | ||
| src_data="$STF_SUITE/tests/functional/cli_root/zfs_receive/zstd_test_data.txt" | ||
| init_data=$TESTDIR/file1 | ||
| write_count=128 | ||
| blksize=131072 | ||
| verify_runnable "global" | ||
| verify_disk_count "$DISKS" 2 | ||
| random_level=$((RANDOM%19 + 1)) | ||
|
|
||
| default_mirror_setup_noexit $DISKS | ||
| log_must zfs set recordsize=$blksize $TESTPOOL/$TESTFS | ||
| log_must zfs set compression=zstd-$random_level $TESTPOOL/$TESTFS | ||
|
|
||
| # write the 1k of text 128 times | ||
| for i in {1..$write_count} | ||
| do | ||
| cat $src_data >> $init_data | ||
| done | ||
|
|
||
| sync_pool $TESTPOOL true | ||
|
|
||
| # get object number of file | ||
| listing=$(ls -i $init_data) | ||
| set -A array $listing | ||
| obj=${array[0]} | ||
| log_note "file $init_data has object number $obj" | ||
|
|
||
| output=$(zdb -Zddddddbbbbbb $TESTPOOL/$TESTFS $obj 2> /dev/null \ | ||
| |grep -m 1 "L0 DVA" |head -n1) | ||
| dva=$(sed -Ene 's/^.+DVA\[0\]=<([^>]+)>.*$/\1/p' <<< "$output") | ||
| log_note "block 0 of $init_data has a DVA of $dva" | ||
|
|
||
| # use the length reported by zdb -ddddddbbbbbb | ||
| size_str=$(sed -Ene 's/^.+ size=([^ ]+) .*$/\1/p' <<< "$output") | ||
| # convert sizes to decimal | ||
| lsize=$(echo $size_str |awk '{split($0,array,"/")} END{print array[1]}') | ||
| lsize_orig=$lsize | ||
| lsize=${lsize%?} | ||
| lsize_bytes=$((16#$lsize)) | ||
| psize=$(echo $size_str |awk '{split($0,array,"/")} END{print array[2]}') | ||
| psize_orig=$psize | ||
| psize=${psize%?} | ||
| psize_bytes=$((16#$psize)) | ||
| log_note "block size $size_str" | ||
|
|
||
| # Get the ZSTD header reported by zdb -Z | ||
| zstd_str=$(sed -Ene 's/^.+ ZSTD:size=([^:]+):version=([^:]+):level=([^:]+):.*$/\1:\2:\3/p' <<< "$output") | ||
| zstd_size=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[1]}') | ||
| log_note "ZSTD compressed size $zstd_size" | ||
| (( $psize_bytes < $zstd_size )) && log_fail \ | ||
| "zdb -Z failed: physical block size was less than header content length ($psize_bytes < $zstd_size)" | ||
|
|
||
| zstd_version=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[2]}') | ||
| log_note "ZSTD version $zstd_version" | ||
|
|
||
| zstd_level=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[3]}') | ||
| log_note "ZSTD level $zstd_level" | ||
| (( $zstd_level != $random_level )) && log_fail \ | ||
| "zdb -Z failed: compression level did not match header level ($zstd_level < $random_level)" | ||
|
|
||
| vdev=$(echo "$dva" |awk '{split($0,array,":")} END{print array[1]}') | ||
| offset=$(echo "$dva" |awk '{split($0,array,":")} END{print array[2]}') | ||
| # Check the first 1024 bytes | ||
| output=$(ZDB_NO_ZLE="true" zdb -R $TESTPOOL $vdev:$offset:$size_str:dr 2> /dev/null) | ||
| outsize=$(wc -c <<< "$output") | ||
| (( $outsize != $blksize )) && log_fail \ | ||
| "zdb -Z failed to decompress the data to the expected length ($outsize != $lsize_bytes)" | ||
| cmp $init_data - <<< "$output" | ||
| (( $? != 0 )) && log_fail "zdb -R :dr failed to decompress the data properly" | ||
|
|
||
| log_pass "zdb -Z flag (ZSTD compression header) works as expected" |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| #!/bin/ksh -p | ||
| # | ||
| # CDDL HEADER START | ||
| # | ||
| # This file and its contents are supplied under the terms of the | ||
| # Common Development and Distribution License ("CDDL"), version 1.0. | ||
| # You may only use this file in accordance with the terms of version | ||
| # 1.0 of the CDDL. | ||
| # | ||
| # A full copy of the text of the CDDL should have accompanied this | ||
| # source. A copy of the CDDL is also available via the Internet at | ||
| # http://www.illumos.org/license/CDDL. | ||
| # | ||
| # CDDL HEADER END | ||
| # | ||
|
|
||
| # | ||
| # Copyright (c) 2020 The FreeBSD Foundation [1] | ||
| # | ||
| # [1] Portions of this software were developed by Allan Jude | ||
| # under sponsorship from the FreeBSD Foundation. | ||
|
|
||
| . $STF_SUITE/include/libtest.shlib | ||
|
|
||
| # | ||
| # DESCRIPTION: | ||
| # ZFS should receive a ZSTD compressed block and be able to determine the level | ||
| # | ||
| # STRATEGY: | ||
| # 1. Create a ZSTD compressed dataset (random level) | ||
| # 2. Create and checksum a file on the compressed dataset | ||
| # 3. Snapshot the compressed dataset | ||
| # 4. Attempt to receive the snapshot into a new dataset | ||
| # 5. Verify the checksum of the file is the same as the original | ||
| # 6. Verify the compression level is correctly stored | ||
| # | ||
|
|
||
| verify_runnable "both" | ||
|
|
||
| function cleanup | ||
| { | ||
| datasetexists $TESTPOOL/$TESTFS1 && \ | ||
| log_must zfs destroy -r $TESTPOOL/$TESTFS1 | ||
|
|
||
| datasetexists $TESTPOOL/$TESTFS2 && \ | ||
| log_must zfs destroy -r $TESTPOOL/$TESTFS2 | ||
| } | ||
|
|
||
| log_onexit cleanup | ||
|
|
||
| log_assert "ZFS should track compression level when receiving a ZSTD stream" | ||
|
|
||
| typeset src_data="$STF_SUITE/tests/functional/cli_root/zfs_receive/zstd_test_data.txt" | ||
| typeset snap="$TESTPOOL/$TESTFS1@snap" | ||
|
|
||
| random_level=$((RANDOM%19 + 1)) | ||
| log_note "Randomly selected ZSTD level: $random_level" | ||
|
|
||
| log_must zfs create -o compress=zstd-$random_level $TESTPOOL/$TESTFS1 | ||
| # Make a 5kb compressible file | ||
| log_must cat $src_data $src_data $src_data $src_data $src_data \ | ||
| > /$TESTPOOL/$TESTFS1/$TESTFILE0 | ||
| typeset checksum=$(md5digest /$TESTPOOL/$TESTFS1/$TESTFILE0) | ||
|
|
||
| log_must zfs snapshot $snap | ||
|
|
||
| # get object number of file | ||
| listing=$(ls -i /$TESTPOOL/$TESTFS1/$TESTFILE0) | ||
| set -A array $listing | ||
| obj=${array[0]} | ||
| log_note "file /$TESTPOOL/$TESTFS1/$TESTFILE0 has object number $obj" | ||
|
|
||
| output=$(zdb -Zddddddbbbbbb $TESTPOOL/$TESTFS1 $obj 2> /dev/null \ | ||
| |grep -m 1 "L0 DVA" |head -n1) | ||
| dva=$(sed -Ene 's/^.+DVA\[0\]=<([^>]+)>.*$/\1/p' <<< "$output") | ||
| log_note "block 0 of /$TESTPOOL/$TESTFS1/$TESTFILE0 has a DVA of $dva" | ||
|
|
||
| zstd_str=$(sed -Ene 's/^.+ ZSTD:size=([^:]+):version=([^:]+):level=([^:]+):.*$/\1:\2:\3/p' <<< "$output") | ||
| zstd_size1=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[1]}') | ||
| zstd_version1=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[2]}') | ||
| zstd_level1=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[3]}') | ||
| log_note "ZSTD src: size=$zstd_size1 version=$zstd_version1 level=$zstd_level1" | ||
|
|
||
| log_note "Verify ZFS can receive the ZSTD compressed stream" | ||
| log_must eval "zfs send -ec $snap | zfs receive $TESTPOOL/$TESTFS2" | ||
|
|
||
| typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0) | ||
| [[ "$cksum1" == "$checksum" ]] || \ | ||
| log_fail "Checksums differ ($cksum1 != $checksum)" | ||
|
|
||
| # get object number of file | ||
| listing=$(ls -i /$TESTPOOL/$TESTFS2/$TESTFILE0) | ||
| set -A array $listing | ||
| obj=${array[0]} | ||
| log_note "file /$TESTPOOL/$TESTFS2/$TESTFILE0 has object number $obj" | ||
|
|
||
| output=$(zdb -Zddddddbbbbbb $TESTPOOL/$TESTFS2 $obj 2> /dev/null \ | ||
| |grep -m 1 "L0 DVA" |head -n1) | ||
| dva=$(sed -Ene 's/^.+DVA\[0\]=<([^>]+)>.*$/\1/p' <<< "$output") | ||
| log_note "block 0 of /$TESTPOOL/$TESTFS2/$TESTFILE0 has a DVA of $dva" | ||
|
|
||
| zstd_str=$(sed -Ene 's/^.+ ZSTD:size=([^:]+):version=([^:]+):level=([^:]+):.*$/\1:\2:\3/p' <<< "$output") | ||
| zstd_size2=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[1]}') | ||
| (( $zstd_size2 != $zstd_size1 )) && log_fail \ | ||
| "ZFS recv failed: compressed size differs ($zstd_size2 != $zstd_size1)" | ||
| zstd_version2=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[2]}') | ||
| zstd_level2=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[3]}') | ||
| log_note "ZSTD dest: size=$zstd_size2 version=$zstd_version2 level=$zstd_level2" | ||
| (( $zstd_level2 != $zstd_level1 )) && log_fail \ | ||
| "ZFS recv failed: compression level did not match header level ($zstd_level2 != $zstd_level1)" | ||
|
|
||
| log_pass "ZFS can receive a ZSTD stream and determine the compression level" |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim.. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| #!/bin/ksh -p | ||
| # | ||
| # CDDL HEADER START | ||
| # | ||
| # The contents of this file are subject to the terms of the | ||
| # Common Development and Distribution License (the "License"). | ||
| # You may not use this file except in compliance with the License. | ||
| # | ||
| # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | ||
| # or http://www.opensolaris.org/os/licensing. | ||
| # See the License for the specific language governing permissions | ||
| # and limitations under the License. | ||
| # | ||
| # When distributing Covered Code, include this CDDL HEADER in each | ||
| # file and include the License file at usr/src/OPENSOLARIS.LICENSE. | ||
| # If applicable, add the following below this CDDL HEADER, with the | ||
| # fields enclosed by brackets "[]" replaced with your own identifying | ||
| # information: Portions Copyright [yyyy] [name of copyright owner] | ||
| # | ||
| # CDDL HEADER END | ||
| # | ||
|
|
||
| # | ||
| # Copyright (c) 2020 The FreeBSD Foundation [1] | ||
| # | ||
| # [1] Portions of this software were developed by Allan Jude | ||
| # under sponsorship from the FreeBSD Foundation. | ||
|
|
||
| . $STF_SUITE/include/libtest.shlib | ||
|
|
||
| # | ||
| # DESCRIPTION: | ||
| # Setting the compression property to any of the zstd levels should activate | ||
| # the zstd feature flag. Destroying the last dataset using the zstd feature flag | ||
| # should revert the feature to the 'enabled' state. | ||
| # | ||
| # STRATEGY: | ||
| # 1. Create pool, then create a file system within it. | ||
| # 2. Check that the zstd feature flag is 'enabled'. | ||
| # 3. Setting the compression property to zstd. | ||
| # 4. Check that the zstd feature flag is now 'active'. | ||
| # 5. Destroy the dataset | ||
| # 6. Confirm that the feature flag reverts to the 'enabled' state. | ||
| # | ||
|
|
||
| verify_runnable "both" | ||
|
|
||
| log_assert "Setting compression=zstd should activate the"\ | ||
| "org.freebsd:zstd_compress feature flag, and destroying the last"\ | ||
| "dataset using that property, should revert the feature flag to"\ | ||
| "the enabled state." | ||
|
|
||
| export VDEV_ZSTD="$TEST_BASE_DIR/vdev-zstd" | ||
|
|
||
| function cleanup | ||
| { | ||
| if poolexists $TESTPOOL-zstd ; then | ||
| destroy_pool $TESTPOOL-zstd | ||
| fi | ||
|
|
||
| rm $VDEV_ZSTD | ||
| } | ||
| log_onexit cleanup | ||
|
|
||
| log_must truncate -s $SPA_MINDEVSIZE $VDEV_ZSTD | ||
| log_must zpool create $TESTPOOL-zstd $VDEV_ZSTD | ||
|
|
||
| featureval="$(get_pool_prop feature@zstd_compress $TESTPOOL-zstd)" | ||
|
|
||
| [[ "$featureval" == "disabled" ]] && \ | ||
| log_unsupported "ZSTD feature flag unsupposed" | ||
|
|
||
| [[ "$featureval" == "active" ]] && \ | ||
| log_unsupported "ZSTD feature already active before test" | ||
|
|
||
| random_level=$((RANDOM%19 + 1)) | ||
| log_note "Randomly selected ZSTD level: $random_level" | ||
|
|
||
| log_must zfs create -o compress=zstd-$random_level $TESTPOOL-zstd/$TESTFS-zstd | ||
|
|
||
| featureval="$(get_pool_prop feature@zstd_compress $TESTPOOL-zstd)" | ||
|
|
||
| log_note "After zfs set, feature flag value is: $featureval" | ||
|
|
||
| [[ "$featureval" == "active" ]] || | ||
| log_fail "ZSTD feature flag not activated" | ||
|
|
||
| log_must zfs destroy $TESTPOOL-zstd/$TESTFS-zstd | ||
|
|
||
| featureval="$(get_pool_prop feature@zstd_compress $TESTPOOL-zstd)" | ||
|
|
||
| log_note "After zfs destroy, feature flag value is: $featureval" | ||
|
|
||
| [[ "$featureval" == "enabled" ]] || | ||
| log_fail "ZSTD feature flag not deactivated" | ||
|
|
||
| log_pass "Setting compression=zstd activated the feature flag, and"\ | ||
| "destroying the dataset deactivated it." |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| #!/bin/ksh -p | ||
| # | ||
| # CDDL HEADER START | ||
| # | ||
| # This file and its contents are supplied under the terms of the | ||
| # Common Development and Distribution License ("CDDL"), version 1.0. | ||
| # You may only use this file in accordance with the terms of version | ||
| # 1.0 of the CDDL. | ||
| # | ||
| # A full copy of the text of the CDDL should have accompanied this | ||
| # source. A copy of the CDDL is also available via the Internet at | ||
| # http://www.illumos.org/license/CDDL. | ||
| # | ||
| # CDDL HEADER END | ||
| # | ||
|
|
||
| # | ||
| # Copyright (c) 2020 The FreeBSD Foundation [1] | ||
| # | ||
| # [1] Portions of this software were developed by Allan Jude | ||
| # under sponsorship from the FreeBSD Foundation. | ||
|
|
||
| . $STF_SUITE/include/libtest.shlib | ||
|
|
||
| export SIZE=1G | ||
| export VDIR=$TESTDIR/disk.persist_l2arc | ||
| export VDEV="$VDIR/a" | ||
| export VDEV_CACHE="$VDIR/b" | ||
| export PASSPHRASE="password" | ||
|
|
||
| # fio options | ||
| export DIRECTORY=/$TESTPOOL-l2arc/encrypted | ||
| export NUMJOBS=4 | ||
| export RUNTIME=30 | ||
| export PERF_RANDSEED=1234 | ||
| export PERF_COMPPERCENT=66 | ||
| export PERF_COMPCHUNK=0 | ||
| export BLOCKSIZE=128K | ||
| export SYNC_TYPE=0 | ||
| export DIRECT=1 | ||
|
|
||
| # | ||
| # DESCRIPTION: | ||
| # System with compressed_arc disabled succeeds at reading from L2ARC | ||
| # | ||
| # STRATEGY: | ||
| # 1. Enable compressed_arc. | ||
| # 2. Create pool with a cache device, encryption, and compression enabled. | ||
| # 3. Read the number of L2ARC checksum failures. | ||
| # 4. Create a random file in that pool and random read for 30 sec. | ||
| # 5. Read the number of L2ARC checksum failures. | ||
| # | ||
|
|
||
| verify_runnable "global" | ||
|
|
||
| log_assert "L2ARC with encryption enabled succeeds." | ||
|
|
||
| origin_carc_setting=$(get_tunable COMPRESSED_ARC_ENABLED) | ||
|
|
||
| function cleanup | ||
| { | ||
| if poolexists $TESTPOOL-l2arc ; then | ||
| destroy_pool $TESTPOOL-l2arc | ||
| fi | ||
|
|
||
| log_must set_tunable64 COMPRESSED_ARC_ENABLED $origin_carc_setting | ||
| } | ||
| log_onexit cleanup | ||
|
|
||
| # Enable Compressed ARC so that in-ARC and on-disk will match | ||
| log_must set_tunable64 COMPRESSED_ARC_ENABLED 1 | ||
|
|
||
| log_must rm -rf $VDIR | ||
| log_must mkdir -p $VDIR | ||
| log_must mkfile $SIZE $VDEV | ||
|
|
||
| typeset fill_mb=800 | ||
| typeset cache_sz=$(( floor($fill_mb / 2) )) | ||
| export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M | ||
|
|
||
| log_must truncate -s ${cache_sz}M $VDEV_CACHE | ||
|
|
||
| log_must zpool create -O compression=zstd -f $TESTPOOL-l2arc $VDEV cache $VDEV_CACHE | ||
|
|
||
| log_must eval "echo $PASSPHRASE | zfs create -o compression=zstd " \ | ||
| "-o encryption=on -o keyformat=passphrase -o keylocation=prompt " \ | ||
| "$TESTPOOL-l2arc/encrypted" | ||
|
|
||
| l2_cksum_bad_start=$(get_arcstat l2_cksum_bad) | ||
|
|
||
| log_must fio $FIO_SCRIPTS/mkfiles.fio | ||
| log_must fio $FIO_SCRIPTS/random_reads.fio | ||
|
|
||
| l2_cksum_bad_end=$(get_arcstat l2_cksum_bad) | ||
|
|
||
| log_note "L2ARC Failed Checksums before: $l2_cksum_bad_start After:"\ | ||
| "$l2_cksum_bad_end" | ||
| log_must test $(( $l2_cksum_bad_end - $l2_cksum_bad_start )) -eq 0 | ||
|
|
||
| log_must zpool destroy -f $TESTPOOL-l2arc | ||
|
|
||
| log_pass "L2ARC with encryption and compressed_arc enabled does not result in"\ | ||
| "checksum errors." |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| #!/bin/ksh -p | ||
| # | ||
| # CDDL HEADER START | ||
| # | ||
| # This file and its contents are supplied under the terms of the | ||
| # Common Development and Distribution License ("CDDL"), version 1.0. | ||
| # You may only use this file in accordance with the terms of version | ||
| # 1.0 of the CDDL. | ||
| # | ||
| # A full copy of the text of the CDDL should have accompanied this | ||
| # source. A copy of the CDDL is also available via the Internet at | ||
| # http://www.illumos.org/license/CDDL. | ||
| # | ||
| # CDDL HEADER END | ||
| # | ||
|
|
||
| # | ||
| # Copyright (c) 2020 The FreeBSD Foundation [1] | ||
| # | ||
| # [1] Portions of this software were developed by Allan Jude | ||
| # under sponsorship from the FreeBSD Foundation. | ||
|
|
||
| . $STF_SUITE/include/libtest.shlib | ||
|
|
||
| export SIZE=1G | ||
| export VDIR=$TESTDIR/disk.persist_l2arc | ||
| export VDEV="$VDIR/a" | ||
| export VDEV_CACHE="$VDIR/b" | ||
| export PASSPHRASE="password" | ||
|
|
||
| # fio options | ||
| export DIRECTORY=/$TESTPOOL-l2arc/encrypted | ||
| export NUMJOBS=4 | ||
| export RUNTIME=30 | ||
| export PERF_RANDSEED=1234 | ||
| export PERF_COMPPERCENT=66 | ||
| export PERF_COMPCHUNK=0 | ||
| export BLOCKSIZE=128K | ||
| export SYNC_TYPE=0 | ||
| export DIRECT=1 | ||
|
|
||
| # | ||
| # DESCRIPTION: | ||
| # System with compressed_arc disabled succeeds at reading from L2ARC | ||
| # | ||
| # STRATEGY: | ||
| # 1. Disable compressed_arc. | ||
| # 2. Create pool with a cache device, encryption, and compression enabled. | ||
| # 3. Read the number of L2ARC checksum failures. | ||
| # 4. Create a random file in that pool and random read for 30 sec. | ||
| # 5. Read the number of L2ARC checksum failures. | ||
| # | ||
|
|
||
| verify_runnable "global" | ||
|
|
||
| log_assert "L2ARC with compressed_arc disabled succeeds." | ||
|
|
||
| origin_carc_setting=$(get_tunable COMPRESSED_ARC_ENABLED) | ||
|
|
||
| function cleanup | ||
| { | ||
| if poolexists $TESTPOOL-l2arc ; then | ||
| destroy_pool $TESTPOOL-l2arc | ||
| fi | ||
|
|
||
| log_must set_tunable64 COMPRESSED_ARC_ENABLED $origin_carc_setting | ||
| } | ||
| log_onexit cleanup | ||
|
|
||
| log_must rm -rf $VDIR | ||
| log_must mkdir -p $VDIR | ||
| log_must mkfile $SIZE $VDEV | ||
|
|
||
| # Disable Compressed ARC so that in-ARC and on-disk will not match | ||
| log_must set_tunable64 COMPRESSED_ARC_ENABLED 0 | ||
|
|
||
| typeset fill_mb=800 | ||
| typeset cache_sz=$(( floor($fill_mb / 2) )) | ||
| export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M | ||
|
|
||
| log_must truncate -s ${cache_sz}M $VDEV_CACHE | ||
|
|
||
| log_must zpool create -O compression=zstd -f $TESTPOOL-l2arc $VDEV cache $VDEV_CACHE | ||
|
|
||
| log_must eval "echo $PASSPHRASE | zfs create -o compression=zstd " \ | ||
| "-o encryption=on -o keyformat=passphrase -o keylocation=prompt " \ | ||
| "$TESTPOOL-l2arc/encrypted" | ||
|
|
||
| l2_cksum_bad_start=$(get_arcstat l2_cksum_bad) | ||
|
|
||
| log_must fio $FIO_SCRIPTS/mkfiles.fio | ||
| log_must fio $FIO_SCRIPTS/random_reads.fio | ||
|
|
||
| l2_cksum_bad_end=$(get_arcstat l2_cksum_bad) | ||
|
|
||
| log_note "L2ARC Failed Checksums before: $l2_cksum_bad_start After:"\ | ||
| "$l2_cksum_bad_end" | ||
| log_must test $(( $l2_cksum_bad_end - $l2_cksum_bad_start )) -eq 0 | ||
|
|
||
| log_must zpool destroy -f $TESTPOOL-l2arc | ||
|
|
||
| log_pass "L2ARC with encryption enabled and compressed_arc disabled does not"\ | ||
| "result in checksum errors." |