From 0b39531d7505ae69bd9a8fbeecad7c6b50460908 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 16 Dec 2020 23:00:35 -0800 Subject: [PATCH 01/11] moving all references to `release` branch was previously `master` --- CONTRIBUTING.md | 4 ++-- README.md | 6 +++--- TESTING.md | 2 +- appveyor.yml | 12 ++++++------ .../zstd_seekable_compression_format.md | 2 +- lib/compress/zstd_compress.c | 2 +- lib/decompress/zstd_decompress_block.c | 2 +- tests/README.md | 2 +- tests/automated_benchmarking.py | 8 ++++---- 9 files changed, 20 insertions(+), 20 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bb85d5876bf..44f2393a2c1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ possible. ## Our Development Process New versions are being developed in the "dev" branch, or in their own feature branch. -When they are deemed ready for a release, they are merged into "master". +When they are deemed ready for a release, they are merged into "release". As a consequences, all contributions must stage first through "dev" or their own feature branch. @@ -383,7 +383,7 @@ CI tests run every time a pull request (PR) is created or updated. The exact tes that get run will depend on the destination branch you specify. Some tests take longer to run than others. Currently, our CI is set up to run a short series of tests when creating a PR to the dev branch and a longer series of tests -when creating a PR to the master branch. You can look in the configuration files +when creating a PR to the release branch. You can look in the configuration files of the respective CI platform for more information on what gets run when. Most people will just want to create a PR with the destination set to their local dev diff --git a/README.md b/README.md index 0f36a5f9d65..dcca7662d2f 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING). ## Contributing -The "dev" branch is the one where all contributions are merged before reaching "master". -If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch. -Direct commit to "master" are not permitted. +The `dev` branch is the one where all contributions are merged before reaching `release`. +If you plan to propose a patch, please commit into the `dev` branch, or its own feature branch. +Direct commit to `release` are not permitted. For more information, please read [CONTRIBUTING](CONTRIBUTING.md). diff --git a/TESTING.md b/TESTING.md index 7e5305178b9..b851d1c8d71 100644 --- a/TESTING.md +++ b/TESTING.md @@ -27,7 +27,7 @@ They consist of the following tests: Long Tests ---------- -Long tests run on all commits to `master` branch, +Long tests run on all commits to `release` branch, and once a day on the current version of `dev` branch, on TravisCI. They consist of the following tests: diff --git a/appveyor.yml b/appveyor.yml index 169c66bd61d..d608f1a3ac5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,13 +1,13 @@ -# Following tests are run _only_ on master branch -# To reproduce these tests, it's possible to push into a branch `appveyorTest` -# or a branch `visual*`, they will intentionnally trigger `master` tests +# Following tests are run _only_ on `release` branch +# and on selected feature branch named `appveyorTest` or `visual*` - version: 1.0.{build} branches: only: + - release - master - - appveyorTest + - /appveyor*/ - /visual*/ environment: matrix: @@ -92,9 +92,9 @@ cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe && appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip && cp zstd.exe ..\bin\zstd.exe && - git clone --depth 1 --branch master https://github.com/facebook/zstd && + git clone --depth 1 --branch release https://github.com/facebook/zstd && cd zstd && - git archive --format=tar master -o zstd-src.tar && + git archive --format=tar release -o zstd-src.tar && ..\zstd -19 zstd-src.tar && appveyor PushArtifact zstd-src.tar.zst && certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig && diff --git a/contrib/seekable_format/zstd_seekable_compression_format.md b/contrib/seekable_format/zstd_seekable_compression_format.md index bf3080f7bbe..55aebfd2e9d 100644 --- a/contrib/seekable_format/zstd_seekable_compression_format.md +++ b/contrib/seekable_format/zstd_seekable_compression_format.md @@ -53,7 +53,7 @@ __`Frame_Size`__ The total size of the skippable frame, not including the `Skippable_Magic_Number` or `Frame_Size`. This is for compatibility with [Zstandard skippable frames]. -[Zstandard skippable frames]: https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#skippable-frames +[Zstandard skippable frames]: https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#skippable-frames #### `Seek_Table_Footer` The seek table footer format is as follows: diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index eb7780cf170..386b051df63 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3258,7 +3258,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, /* Dictionary format : * See : - * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format */ /*! ZSTD_loadZstdDictionary() : * @return : dictID, or an error code diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index bec82e85709..19cbdc5c16e 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -236,7 +236,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, /* Default FSE distribution tables. * These are pre-calculated FSE decoding tables using default distributions as defined in specification : - * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#default-distributions + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions * They were generated programmatically with following method : * - start from default distributions, present in /lib/common/zstd_internal.h * - generate tables normally, using ZSTD_buildFSETable() diff --git a/tests/README.md b/tests/README.md index 23e00767c39..1e40c46aae9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -28,7 +28,7 @@ desktop machine for every pull request that is made to the zstd repo but can als be run on any machine via the command line interface. There are three modes of usage for this script: fastmode will just run a minimal single -build comparison (between facebook:dev and facebook:master), onetime will pull all the current +build comparison (between facebook:dev and facebook:release), onetime will pull all the current pull requests from the zstd repo and compare facebook:dev to all of them once, continuous will continuously get pull requests from the zstd repo and run benchmarks against facebook:dev. diff --git a/tests/automated_benchmarking.py b/tests/automated_benchmarking.py index d0cfb1fbe37..77eea29de27 100644 --- a/tests/automated_benchmarking.py +++ b/tests/automated_benchmarking.py @@ -20,7 +20,7 @@ GITHUB_API_PR_URL = "https://api.github.com/repos/facebook/zstd/pulls?state=open" GITHUB_URL_TEMPLATE = "https://github.com/{}/zstd" -MASTER_BUILD = {"user": "facebook", "branch": "dev", "hash": None} +RELEASE_BUILD = {"user": "facebook", "branch": "dev", "hash": None} # check to see if there are any new PRs every minute DEFAULT_MAX_API_CALL_FREQUENCY_SEC = 60 @@ -264,11 +264,11 @@ def main(filenames, levels, iterations, builds=None, emails=None, continuous=Fal for test_build in builds: if dictionary_filename == None: regressions = get_regressions( - MASTER_BUILD, test_build, iterations, filenames, levels + RELEASE_BUILD, test_build, iterations, filenames, levels ) else: regressions = get_regressions_dictionary( - MASTER_BUILD, test_build, filenames, dictionary_filename, levels, iterations + RELEASE_BUILD, test_build, filenames, dictionary_filename, levels, iterations ) body = "\n".join(regressions) if len(regressions) > 0: @@ -320,7 +320,7 @@ def main(filenames, levels, iterations, builds=None, emails=None, continuous=Fal builds = [{"user": None, "branch": "None", "hash": None}] main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename) elif mode == "fastmode": - builds = [{"user": "facebook", "branch": "master", "hash": None}] + builds = [{"user": "facebook", "branch": "release", "hash": None}] main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename) else: main(filenames, levels, iterations, None, emails, True, frequency=frequency, dictionary_filename=dictionary_filename) From ce34dc39a07661df751a0a5792aed28bf47dbcac Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 16 Dec 2020 23:53:13 -0800 Subject: [PATCH 02/11] removed incorrect test --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index d608f1a3ac5..0e872557525 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,7 +14,7 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests fullbench-lib" + SCRIPT: "make allzstd MOREFLAGS=-static" ARTIFACT: "true" BUILD: "true" - COMPILER: "gcc" From ae85676d44baee3d12168a5c929347b3836f2cf2 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 17 Dec 2020 14:27:53 -0800 Subject: [PATCH 03/11] Fix alignment of scratchBuffer in HUF_compressWeights() The scratch buffer must be 4-byte aligned. This causes test failures in 32-bit systems, where the stack isn't aligned. Fixes Issue #2428. --- lib/common/fse.h | 5 +++-- lib/compress/huf_compress.c | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/common/fse.h b/lib/common/fse.h index 83a07847aaa..dd5fc44e809 100644 --- a/lib/common/fse.h +++ b/lib/common/fse.h @@ -335,9 +335,10 @@ size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); /* FSE_buildCTable_wksp() : * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). - * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog)`. + * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. */ -#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * (maxSymbolValue + 2) + (1ull << tableLog)) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (maxSymbolValue + 2 + (1ull << (tableLog - 2))) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); #define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index abbcc319224..00c593d7ed2 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -69,7 +69,7 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; - BYTE scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; + U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; unsigned count[HUF_TABLELOG_MAX+1]; S16 norm[HUF_TABLELOG_MAX+1]; From 4680d817c0522c57ad905aadef1bb7e1b59bf0f5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 17 Dec 2020 14:53:36 -0800 Subject: [PATCH 04/11] added a simple runtime test in CI for 32-bit binaries --- .github/workflows/generic-dev.yml | 14 ++++++++++---- .github/workflows/generic-release.yml | 4 ++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml index bb88de57ca5..3f564878b6f 100644 --- a/.github/workflows/generic-dev.yml +++ b/.github/workflows/generic-dev.yml @@ -2,14 +2,13 @@ name: generic-dev on: pull_request: - branches: [ dev, master, actionsTest ] + branches: [ dev, release, actionsTest ] jobs: # Dev PR jobs that still have to be migrated from travis # -# icc (need self-hosted) -# versionTag +# versionTag (only on release tags) # valgrindTest (keeps failing for some reason. need investigation) # staticAnalyze (need trusty so need self-hosted) # pcc-fuzz: (need trusty so need self-hosted) @@ -19,7 +18,7 @@ jobs: # I need admins permissions to the repo for that it looks like # So I'm tabling that for now # -# The master branch exclusive jobs will be in a separate +# The release branch exclusive jobs will be in a separate # workflow file (the osx tests and meson build that is) benchmarking: @@ -36,6 +35,13 @@ jobs: - name: make test run: make test + check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: make check on 32-bit + run: CFLAGS="-m32 -O1 -fstack-protector" make test V=1 + gcc-6-7-libzstd: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/generic-release.yml b/.github/workflows/generic-release.yml index de4a1cb9145..a41f80f3e00 100644 --- a/.github/workflows/generic-release.yml +++ b/.github/workflows/generic-release.yml @@ -2,10 +2,10 @@ name: generic-release on: pull_request: - # This will eventually only be for pushes to master + # This will eventually only be for pushes to release # but for dogfooding purposes, I'm running it even # on dev pushes - branches: [ dev, master, actionsTest ] + branches: [ dev, release, actionsTest ] jobs: # missing jobs From c11db9c8b5dcf2bd03f507304f8f081cb9afe4cb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 17 Dec 2020 15:01:04 -0800 Subject: [PATCH 05/11] additional master->release switches (CI tests) --- .circleci/config.yml | 3 ++- .github/workflows/linux-kernel.yml | 2 +- .travis.yml | 23 ++++++++++++----------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bcf2e1d5ea2..9738c1bd45b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -143,8 +143,9 @@ workflows: filters: branches: only: - - master + - release - dev + - master jobs: # Run daily long regression tests - regression-test diff --git a/.github/workflows/linux-kernel.yml b/.github/workflows/linux-kernel.yml index 35871ff0850..124f7777828 100644 --- a/.github/workflows/linux-kernel.yml +++ b/.github/workflows/linux-kernel.yml @@ -2,7 +2,7 @@ name: linux-kernel on: pull_request: - branches: [ dev, master, actionsTest ] + branches: [ dev, release, actionsTest ] jobs: test: diff --git a/.travis.yml b/.travis.yml index 226d4c0154e..ee473733c2b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ git: branches: only: - dev + - release - master - travisTest @@ -162,33 +163,33 @@ matrix: - make -C tests checkTag - tests/checkTag "$TRAVIS_BRANCH" - # tests for master branch and cron job only + # tests for release branch and cron job only - name: OS-X # ~13mn - if: branch = master + if: branch = release os: osx script: - make test - make -C lib all - name: zbuff test - if: branch = master + if: branch = release script: - make -C tests test-zbuff - name: Versions Compatibility Test # 11.5mn - if: branch = master + if: branch = release script: - make -C tests versionsTest - name: thread sanitizer # ~29mn - if: branch = master + if: branch = release script: - make clang38install - CC=clang-3.8 make tsan-test-zstream - CC=clang-3.8 make tsan-fuzztest - name: PPC64LE + Fuzz test # ~13mn - if: branch = master + if: branch = release arch: ppc64le script: - cat /proc/cpuinfo @@ -196,28 +197,28 @@ matrix: - name: Qemu PPC64 + Fuzz test # ~13mn, presumed Big-Endian (?) dist: trusty # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial - if: branch = master + if: branch = release script: - make ppcinstall - make ppc64fuzz # note : we already have aarch64 tests on hardware - name: Qemu aarch64 + Fuzz Test (on Xenial) # ~14mn - if: branch = master + if: branch = release dist: xenial script: - make arminstall - make aarch64fuzz - name: zlib wrapper test # ~7.5mn - if: branch = master + if: branch = release script: - make gpp6install valgrindinstall - make -C zlibWrapper test - make -C zlibWrapper valgrindTest - name: LZ4, thread pool, and partial libs tests # ~4mn - if: branch = master + if: branch = release script: - make lz4install - make -C tests test-lz4 @@ -229,7 +230,7 @@ matrix: # meson dedicated test - name: Xenial (Meson + clang) # ~15mn - if: branch = master + if: branch = release dist: xenial language: cpp compiler: clang From d5eb7d156904e2c7ffbfc3fd71285ac1340792b2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 17 Dec 2020 15:05:26 -0800 Subject: [PATCH 06/11] added pre-requisites for 32-bit tests in CI --- .github/workflows/generic-dev.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml index 3f564878b6f..4f0a677be1c 100644 --- a/.github/workflows/generic-dev.yml +++ b/.github/workflows/generic-dev.yml @@ -40,7 +40,9 @@ jobs: steps: - uses: actions/checkout@v2 - name: make check on 32-bit - run: CFLAGS="-m32 -O1 -fstack-protector" make test V=1 + run: | + make libc6install + CFLAGS="-m32 -O1 -fstack-protector" make test V=1 gcc-6-7-libzstd: runs-on: ubuntu-latest From 3536e9d5ff659c7804214aeed599f17cd2666e4c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 17 Dec 2020 15:44:54 -0800 Subject: [PATCH 07/11] removing tests using too much resources for 32-bit address space --- .github/workflows/generic-dev.yml | 2 +- tests/playTests.sh | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml index 4f0a677be1c..9abe98b73ab 100644 --- a/.github/workflows/generic-dev.yml +++ b/.github/workflows/generic-dev.yml @@ -42,7 +42,7 @@ jobs: - name: make check on 32-bit run: | make libc6install - CFLAGS="-m32 -O1 -fstack-protector" make test V=1 + CFLAGS="-m32 -O1 -fstack-protector" make check V=1 gcc-6-7-libzstd: runs-on: ubuntu-latest diff --git a/tests/playTests.sh b/tests/playTests.sh index 51b42b60ac4..4d6abbcc066 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1342,8 +1342,6 @@ optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) -optCSize22=$(datagen -g900K | zstd -22 --ultra -c | wc -c) -longCSize22=$(datagen -g900K | zstd -22 --ultra --long -c | wc -c) if [ "$longCSize16" -gt "$optCSize16" ]; then echo using --long on compression level 16 should not cause compressed size regression exit 1 @@ -1353,9 +1351,6 @@ elif [ "$longCSize19" -gt "$optCSize19" ]; then elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then echo using --long on compression level 19 with wLog=23 should not cause compressed size regression exit 1 -elif [ "$longCSize22" -gt "$optCSize22" ]; then - echo using --long on compression level 22 should not cause compressed size regression - exit 1 fi From 54cc01c5be9fa9857454cda737980b1376c33797 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 18 Dec 2020 11:01:22 -0800 Subject: [PATCH 08/11] removed tests duplicated between TravisCI and Github Actions reduce load on TravisCI --- .github/workflows/generic-dev.yml | 10 +++- .travis.yml | 78 ------------------------------- 2 files changed, 9 insertions(+), 79 deletions(-) diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml index 9abe98b73ab..8882fcc0b6a 100644 --- a/.github/workflows/generic-dev.yml +++ b/.github/workflows/generic-dev.yml @@ -58,6 +58,10 @@ jobs: LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt make -C tests zbufftest-dll + # candidate test (to check) : underlink test + # LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked + # zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread + gcc-8-asan-ubsan-testzstd: runs-on: ubuntu-16.04 # fails on 18.04 steps: @@ -92,6 +96,10 @@ jobs: sudo apt-get install clang-3.8 CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 + # Note : external libraries must be turned off when using MSAN tests, + # because they are not msan-instrumented, + # so any data coming from these libraries is always considered "uninitialized" + cmake-build-and-test-check: runs-on: ubuntu-latest steps: @@ -155,7 +163,7 @@ jobs: make clean make c99build make clean - make travis-install + make travis-install # just ensures `make install` works mingw-cross-compilation: runs-on: ubuntu-latest diff --git a/.travis.yml b/.travis.yml index ee473733c2b..fd4b29e377c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,49 +32,11 @@ matrix: script: - make check - - name: make benchmarking - script: - - make benchmarking - - name: make test (complete) script: # DEVNULLRIGHTS : will request sudo rights to test permissions on /dev/null - DEVNULLRIGHTS=test make test - - name: gcc-6 + gcc-7 + libzstdmt compilation # ~ 6mn - script: - - make gcc6install gcc7install - - CC=gcc-6 CFLAGS=-Werror make -j all - - make clean - - CC=gcc-7 CFLAGS=-Werror make -j all - - make clean - - LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt - - make -C tests zbufftest-dll - # LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked - # zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread - - - name: gcc-8 + ASan + UBSan + Test Zstd # ~6.5mn - script: - - make gcc8install - - CC=gcc-8 CFLAGS="-Werror" make -j all - - make clean - - CC=gcc-8 make -j uasan-test-zstd Date: Fri, 18 Dec 2020 12:50:36 -0800 Subject: [PATCH 09/11] removed duplicated release-only tests from travisCI as they are already part of Github Actions --- .github/workflows/generic-release.yml | 1 + .travis.yml | 23 ----------------------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/.github/workflows/generic-release.yml b/.github/workflows/generic-release.yml index a41f80f3e00..167ba38487f 100644 --- a/.github/workflows/generic-release.yml +++ b/.github/workflows/generic-release.yml @@ -44,6 +44,7 @@ jobs: sudo apt-get install clang-3.8 CC=clang-3.8 make tsan-test-zstream CC=clang-3.8 make tsan-fuzztest + zlib-wrapper: runs-on: ubuntu-16.04 steps: diff --git a/.travis.yml b/.travis.yml index fd4b29e377c..4757c6b02ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -93,11 +93,6 @@ matrix: - make test - make -C lib all - - name: zbuff test - if: branch = release - script: - - make -C tests test-zbuff - - name: Versions Compatibility Test # 11.5mn if: branch = release script: @@ -132,24 +127,6 @@ matrix: - make arminstall - make aarch64fuzz - - name: zlib wrapper test # ~7.5mn - if: branch = release - script: - - make gpp6install valgrindinstall - - make -C zlibWrapper test - - make -C zlibWrapper valgrindTest - - - name: LZ4, thread pool, and partial libs tests # ~4mn - if: branch = release - script: - - make lz4install - - make -C tests test-lz4 - - make check < /dev/null | tee # mess with lz4 console detection - - make clean - - make -C tests test-pool - - make clean - - bash tests/libzstd_partial_builds.sh - # meson dedicated test - name: Xenial (Meson + clang) # ~15mn if: branch = release From a7cb4af573ffdac0ba9c9ddca60cf70ffd0e9b31 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 18 Dec 2020 15:04:09 -0800 Subject: [PATCH 10/11] added emphasis on the alignment condition of workspace and made it a programming mistake (`assert()`) rather than a runtime error. --- lib/compress/huf_compress.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 00c593d7ed2..302e08864da 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -736,25 +736,26 @@ typedef struct { } HUF_compress_tables_t; /* HUF_compress_internal() : - * `workSpace` must a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ + * `workSpace_align4` must be aligned on 4-bytes boundaries, + * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U32 unsigned */ static size_t HUF_compress_internal (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, HUF_nbStreams_e nbStreams, - void* workSpace, size_t wkspSize, + void* workSpace_align4, size_t wkspSize, HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat, const int bmi2) { - HUF_compress_tables_t* const table = (HUF_compress_tables_t*)workSpace; + HUF_compress_tables_t* const table = (HUF_compress_tables_t*)workSpace_align4; BYTE* const ostart = (BYTE*)dst; BYTE* const oend = ostart + dstSize; BYTE* op = ostart; HUF_STATIC_ASSERT(sizeof(*table) <= HUF_WORKSPACE_SIZE); + assert(((size_t)workSpace_align4 & 3) == 0); /* must be aligned on 4-bytes boundaries */ /* checks & inits */ - if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ if (wkspSize < HUF_WORKSPACE_SIZE) return ERROR(workSpace_tooSmall); if (!srcSize) return 0; /* Uncompressed */ if (!dstSize) return 0; /* cannot fit anything within dst budget */ @@ -772,7 +773,7 @@ HUF_compress_internal (void* dst, size_t dstSize, } /* Scan input and build symbol stats */ - { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace, wkspSize) ); + { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace_align4, wkspSize) ); if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ } From 7c495e8ea2fd8ef39084c44e24c2f854aff2b68e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 18 Dec 2020 15:52:11 -0800 Subject: [PATCH 11/11] updated version number to v1.4.8 --- CHANGELOG | 5 ++++- doc/zstd_manual.html | 4 ++-- lib/zstd.h | 2 +- programs/zstd.1 | 2 +- programs/zstdgrep.1 | 2 +- programs/zstdless.1 | 2 +- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 562211cfdee..86092563177 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,7 @@ -v1.4.7 +v1.4.8 (Dec 18, 2020) +hotfix: wrong alignment of an internal buffer + +v1.4.7 (Dec 16, 2020) perf: stronger --long mode at high compression levels, by @senhuang42 perf: stronger --patch-from at high compression levels, thanks to --long improvements perf: faster dictionary compression at medium compression levels, by @felixhandte diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index cb5ded0703a..315488844ba 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1,10 +1,10 @@ -zstd 1.4.7 Manual +zstd 1.4.8 Manual -

zstd 1.4.7 Manual

+

zstd 1.4.8 Manual

Contents

    diff --git a/lib/zstd.h b/lib/zstd.h index 06e07f7c34d..b0ecdf55385 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -72,7 +72,7 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 7 +#define ZSTD_VERSION_RELEASE 8 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) /*! ZSTD_versionNumber() : diff --git a/programs/zstd.1 b/programs/zstd.1 index 0335b175cee..a8fc277672b 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -1,5 +1,5 @@ . -.TH "ZSTD" "1" "December 2020" "zstd 1.4.7" "User Commands" +.TH "ZSTD" "1" "December 2020" "zstd 1.4.8" "User Commands" . .SH "NAME" \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files diff --git a/programs/zstdgrep.1 b/programs/zstdgrep.1 index c8af908c984..4d143b59873 100644 --- a/programs/zstdgrep.1 +++ b/programs/zstdgrep.1 @@ -1,5 +1,5 @@ . -.TH "ZSTDGREP" "1" "December 2020" "zstd 1.4.7" "User Commands" +.TH "ZSTDGREP" "1" "December 2020" "zstd 1.4.8" "User Commands" . .SH "NAME" \fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files diff --git a/programs/zstdless.1 b/programs/zstdless.1 index be92e3514ec..43f235453ee 100644 --- a/programs/zstdless.1 +++ b/programs/zstdless.1 @@ -1,5 +1,5 @@ . -.TH "ZSTDLESS" "1" "December 2020" "zstd 1.4.7" "User Commands" +.TH "ZSTDLESS" "1" "December 2020" "zstd 1.4.8" "User Commands" . .SH "NAME" \fBzstdless\fR \- view zstandard\-compressed files