Added Zstandard support for zip archives and CHDs. (mamedev#11827)

* 3rdparty/zstd: Added Zstandard compression library version 1.5.5. * util/unzip.cpp: Added support for Zstandard compression (method 93). * util/chdcodec.cpp: Added support for Zstandard compression. * 3rdparty/flac: Always define NDEBUG to avoid log spam.
Luigi30 · Dec 16, 2023 · 75aa660 · 75aa660
1 parent f593a07
commit 75aa660
Show file tree

Hide file tree

Showing 607 changed files with 163,840 additions and 17 deletions.
diff --git a/3rdparty/README.md b/3rdparty/README.md
@@ -22,12 +22,12 @@ dxsdk - [DirectX SDK EULA](https://github.com/mamedev/mame/blob/master/3rdparty/
 
 expat - [The MIT License (MIT)](http://opensource.org/licenses/MIT)
 
+flac - [The BSD 3-Clause License](http://opensource.org/licenses/BSD-3-Clause) (codecs), [The GNU General Public License, version 2](https://opensource.org/licenses/GPL-2.0)
+
 genie - [The BSD 3-Clause License](http://opensource.org/licenses/BSD-3-Clause)
 
 glm - [The MIT License (MIT)](http://opensource.org/licenses/MIT)
 
-libflac - [The BSD 3-Clause License](http://opensource.org/licenses/BSD-3-Clause) (codecs), [The GNU General Public License, version 2](https://opensource.org/licenses/GPL-2.0)
-
 libjpeg - [Custom BSD-like](https://github.com/numenta/nupic/blob/master/external/licenses/LICENSE.libjpeg-6b.txt)
 
 linenoise - [The BSD 2-Clause License](http://opensource.org/licenses/BSD-2-Clause)
@@ -71,3 +71,5 @@ utf8proc - [The MIT License (MIT)](http://opensource.org/licenses/MIT)
 wdlfft - [zlib license](http://opensource.org/licenses/Zlib)
 
 zlib - [zlib license](http://opensource.org/licenses/Zlib)
+
+zstd - Dual-licensed [The BSD 3-Clause License](http://opensource.org/licenses/BSD-3-Clause) and [The GNU General Public License, version 2](https://opensource.org/licenses/GPL-2.0)
diff --git a/3rdparty/zstd/.buckconfig b/3rdparty/zstd/.buckconfig
@@ -0,0 +1,9 @@
+[cxx]
+  cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
+  cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith
+  cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
+  cxxflags = -std=c++11 -Wno-deprecated-declarations
+  gtest_dep = //contrib/pzstd:gtest
+
+[httpserver]
+  port = 0
diff --git a/3rdparty/zstd/.buckversion b/3rdparty/zstd/.buckversion
@@ -0,0 +1 @@
+c8dec2e8da52d483f6dd7c6cd2ad694e8e6fed2b
diff --git a/3rdparty/zstd/.circleci/config.yml b/3rdparty/zstd/.circleci/config.yml
@@ -0,0 +1,123 @@
+version: 2
+
+jobs:
+  # the first half of the jobs are in this test
+  short-tests-0:
+    # TODO: Create a small custom docker image with all the dependencies we need
+    #       preinstalled to reduce installation time.
+    docker:
+      - image: fbopensource/zstd-circleci-primary:0.0.1
+    steps:
+      - checkout
+      - run:
+          name: Test
+          command: |
+            ./tests/test-license.py
+            cc -v
+            CFLAGS="-O0 -Werror -pedantic" make allmost; make clean
+            make c99build; make clean
+            make c11build; make clean
+            make -j regressiontest; make clean
+            make shortest; make clean
+            make cxxtest; make clean
+  # the second half of the jobs are in this test
+  short-tests-1:
+    docker:
+      - image: fbopensource/zstd-circleci-primary:0.0.1
+    steps:
+      - checkout
+      - run:
+          name: Test
+          command: |
+            make gnu90build; make clean
+            make gnu99build; make clean
+            make ppc64build V=1; make clean
+            make ppcbuild   V=1; make clean
+            make armbuild   V=1; make clean
+            make aarch64build V=1; make clean
+            make -C tests test-legacy test-longmatch; make clean
+            make -C lib libzstd-nomt; make clean
+  # This step should only be run in a cron job
+  regression-test:
+    docker:
+      - image: fbopensource/zstd-circleci-primary:0.0.1
+    environment:
+      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+    steps:
+      - checkout
+      # Restore the cached resources.
+      - restore_cache:
+          # We try our best to bust the cache when the data changes by hashing
+          # data.c. If that doesn't work, simply update the version number here
+          # and below. If we fail to bust the cache, the regression testing will
+          # still work, since it has its own stamp, but will need to redownload
+          # everything.
+          keys:
+            - regression-cache-{{ checksum "tests/regression/data.c" }}-v0
+      - run:
+          name: Regression Test
+          command: |
+            make -C programs zstd
+            make -C tests/regression test
+            mkdir -p $CIRCLE_ARTIFACTS
+            ./tests/regression/test                     \
+                --cache  tests/regression/cache         \
+                --output $CIRCLE_ARTIFACTS/results.csv  \
+                --zstd   programs/zstd
+            echo "NOTE: The new results.csv is uploaded as an artifact to this job"
+            echo "      If this fails, go to the Artifacts pane in CircleCI, "
+            echo "      download /tmp/circleci-artifacts/results.csv, and if they "
+            echo "      are still good, copy it into the repo and commit it."
+            echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
+            diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
+      # Only save the cache on success (default), since if the failure happened
+      # before we stamp the data cache, we will have a bad cache for this key.
+      - save_cache:
+          key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
+          paths:
+            - tests/regression/cache
+      - store_artifacts:
+          path: /tmp/circleci-artifacts
+
+
+workflows:
+  version: 2
+  commit:
+    jobs:
+      # Run the tests in parallel
+      - short-tests-0
+      - short-tests-1
+      - regression-test
+
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - release
+                - dev
+                - master
+    jobs:
+      # Run daily regression tests
+      - regression-test
+
+
+
+  # Longer tests
+    #- make -C tests test-zstd-nolegacy && make clean
+    #- pyenv global 3.4.4; make -C tests versionsTest && make clean
+    #- make zlibwrapper         && make clean
+    #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
+    #- make uasan               && make clean
+    #- make asan32              && make clean
+    #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
+  # Valgrind tests
+    #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
+    #- make -C tests valgrindTest && make clean
+  # ARM, AArch64, PowerPC, PowerPC64 tests
+    #- make ppctest             && make clean
+    #- make ppc64test           && make clean
+    #- make armtest             && make clean
+    #- make aarch64test         && make clean
diff --git a/3rdparty/zstd/.circleci/images/primary/Dockerfile b/3rdparty/zstd/.circleci/images/primary/Dockerfile
@@ -0,0 +1,9 @@
+FROM circleci/buildpack-deps@sha256:f6f10c11b7b8ccfd4f4a5b830c3256803604ce61292b60cb22e26b12f62b0e8c
+
+RUN sudo dpkg --add-architecture i386
+RUN sudo apt-get -y -qq update
+RUN sudo apt-get -y install \
+    gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
+    libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
+    libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
+    libcurl4-openssl-dev
diff --git a/3rdparty/zstd/.cirrus.yml b/3rdparty/zstd/.cirrus.yml
@@ -0,0 +1,10 @@
+task:
+  name: FreeBSD (shortest)
+  freebsd_instance:
+    matrix:
+      image_family: freebsd-13-0
+      image_family: freebsd-12-2
+  install_script: pkg install -y gmake coreutils
+  script: |
+    MOREFLAGS="-Werror" gmake -j all
+    gmake shortest
diff --git a/3rdparty/zstd/.github/ISSUE_TEMPLATE/bug_report.md b/3rdparty/zstd/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,35 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Downloads data '...'
+2. Run '...' with flags '...'
+3. Scroll up on the log to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots and charts**
+If applicable, add screenshots and charts to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. Mac]
+ - Version [e.g. 22]
+ - Compiler [e.g. gcc]
+ - Flags [e.g. O2]
+ - Other relevant hardware specs [e.g. Dual-core]
+ - Build system [e.g. Makefile]
+
+**Additional context**
+Add any other context about the problem here.
diff --git a/3rdparty/zstd/.github/ISSUE_TEMPLATE/feature_request.md b/3rdparty/zstd/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.
diff --git a/3rdparty/zstd/.github/dependabot.yml b/3rdparty/zstd/.github/dependabot.yml
@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"