355 changes: 355 additions & 0 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,355 @@
# Copyright (c) the JPEG XL Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# Workflow for building the release binaries.
#
# This workflow runs as a post-submit step, when pushing to main or the release
# branches (v*.*.x), and when creating a release in GitHub.
#
# In the GitHub release case, in addition to build the release binaries it also
# uploads the binaries to the given release automatically.

name: Release build / deploy
on:
push:
branches:
- main
- v*.*.x
release:
types: [ published ]

jobs:
ubuntu_static_x86_64:
name: Release linux x86_64 static
runs-on: [ubuntu-latest]
steps:
- name: Install build deps
run: |
sudo apt update
sudo apt install -y \
asciidoc \
clang \
cmake \
doxygen \
libbrotli-dev \
libgdk-pixbuf2.0-dev \
libgif-dev \
libgtest-dev \
libgtk2.0-dev \
libjpeg-dev \
libopenexr-dev \
libpng-dev \
libwebp-dev \
ninja-build \
pkg-config \
#
echo "CC=clang" >> $GITHUB_ENV
echo "CXX=clang++" >> $GITHUB_ENV
- name: Checkout the source
uses: actions/checkout@v2
with:
submodules: true
fetch-depth: 1

- name: Build
env:
SKIP_TEST: 1
run: |
./ci.sh release \
-DJPEGXL_DEP_LICENSE_DIR=/usr/share/doc \
-DJPEGXL_STATIC=ON \
-DBUILD_TESTING=OFF \
-DJPEGXL_ENABLE_VIEWERS=OFF \
-DJPEGXL_ENABLE_PLUGINS=OFF \
-DJPEGXL_ENABLE_OPENEXR=OFF \
- name: Package release tarball
run: |
cd build
tar -zcvf ${{ runner.workspace }}/release_file.tar.gz \
LICENSE* tools/{cjxl,djxl,benchmark_xl}
- name: Upload artifacts
uses: actions/upload-artifact@v2
with:
name: jxl-linux-x86_64-static
path: ${{ runner.workspace }}/release_file.tar.gz

- name: Upload binaries to release
if: github.event_name == 'release'
uses: svenstaro/upload-release-action@v1-release
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: ${{ runner.workspace }}/release_file.tar.gz
asset_name: jxl-linux-x86_64-static-${{ github.event.release.tag_name }}.tar.gz
tag: ${{ github.ref }}
overwrite: true


# Build .deb packages Ubuntu/Debian
release_ubuntu_pkg:
name: .deb packages / ${{ matrix.os }}
runs-on: ubuntu-latest
strategy:
matrix:
os:
- ubuntu:20.04
- ubuntu:18.04
- debian:buster
- debian:bullseye
- debian:bookworm
- debian:sid

container:
image: ${{ matrix.os }}

steps:
- name: Set env
shell: 'bash'
id: 'env'
run: |
artifact_name="jxl-debs-amd64-${matrix_os/:/-}"
echo ${artifact_name}
echo "::set-output name=artifact_name::${artifact_name}"
env:
matrix_os: ${{ matrix.os }}

- name: Install build deps
run: |
apt update
DEBIAN_FRONTEND=noninteractive apt install -y \
build-essential \
devscripts \
#
- name: Install git (only 18.04)
if: matrix.os == 'ubuntu:18.04'
# Ubuntu 18.04 ships with git 2.17 but we need 2.18 or newer for
# actions/checkout@v2 to work
shell: 'bash'
run: |
apt install -y \
libcurl4-openssl-dev \
libexpat1-dev \
libssl-dev \
wget \
zlib1g-dev \
#
git_version="2.32.0"
wget -nv \
"https://github.com/git/git/archive/refs/tags/v${git_version}.tar.gz"
tar -zxf "v${git_version}.tar.gz"
cd "git-${git_version}"
make prefix=/usr -j4 install
- name: Checkout the source
uses: actions/checkout@v2
with:
submodules: true
fetch-depth: 1

- name: Stamp non-release versions
# Stamps the built package with the commit date as part of the version
# after the version number so newer release candidates can override older
# ones.
if: github.event_name != 'release'
shell: 'bash'
run: |
# Committer timestamp.
set -x
commit_timestamp=$(git show -s --format=%ct)
commit_datetime=$(date --utc "--date=@${commit_timestamp}" '+%Y%m%d%H%M%S')
commit_ref=$(git rev-parse --short HEAD)
sem_version=$(dpkg-parsechangelog --show-field Version)
sem_version="${sem_version%%-*}"
deb_version="${sem_version}~alpha${commit_datetime}-0+git${commit_ref}"
dch -M --distribution unstable -b --newversion "${deb_version}" \
"Stamping build with version ${deb_version}"
- name: Stamp release versions
# Mark the version as released
if: github.event_name == 'release'
shell: 'bash'
run: |
if head -n1 debian/changelog | grep UNRELEASED; then
dch -M --distribution unstable --release ''
fi
- name: Install gtest (only 18.04)
if: matrix.os == 'ubuntu:18.04'
# In Ubuntu 18.04 no package installed the libgtest.a. libgtest-dev
# installs the source files only.
run: |
apt install -y libgtest-dev cmake
for prj in googletest googlemock; do
(cd /usr/src/googletest/${prj}/ &&
cmake CMakeLists.txt -DCMAKE_INSTALL_PREFIX=/usr &&
make all install)
done
# Remove libgmock-dev dependency in Ubuntu 18.04. It doesn't exist there.
sed '/libgmock-dev,/d' -i debian/control
- name: Remove libjxl-gimp-plugin package (only 18.04)
if: matrix.os == 'ubuntu:18.04'
run: |
# Gimp 2.8 is not supported.
sed -i '/Package: libjxl-gimp-plugin/,/^$/d' debian/control
- name: Build hwy
run: |
apt build-dep -y ./third_party/highway
./ci.sh debian_build highway
dpkg -i build/debs/libhwy-dev_*_amd64.deb
- name: Build libjxl
run: |
apt build-dep -y .
./ci.sh debian_build jpeg-xl
- name: Stats
run: |
./ci.sh debian_stats
- name: Upload artifacts
uses: actions/upload-artifact@v2
with:
name: ${{ steps.env.outputs.artifact_name }}
path: |
build/debs/*jxl*.*
- name: Package release tarball
if: github.event_name == 'release'
run: |
(cd build/debs/; find -maxdepth 1 -name '*jxl*.*') | \
tar -zcvf release_file.tar.gz -C build/debs/ -T -
- name: Upload binaries to release
if: github.event_name == 'release'
uses: svenstaro/upload-release-action@v1-release
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: release_file.tar.gz
asset_name: ${{ steps.env.outputs.artifact_name }}-${{ github.event.release.tag_name }}.tar.gz
tag: ${{ github.ref }}
overwrite: true


windows_build:
name: Windows Build (vcpkg / ${{ matrix.triplet }})
runs-on: [windows-latest]
strategy:
matrix:
include:
- triplet: x86-windows-static
arch: '-A Win32'
- triplet: x64-windows-static
arch: '-A x64'

env:
VCPKG_VERSION: '2021.05.12'
VCPKG_ROOT: vcpkg
VCPKG_DISABLE_METRICS: 1

steps:
- name: Checkout the source
uses: actions/checkout@v2
with:
submodules: true
fetch-depth: 2

- uses: actions/cache@v2
id: cache-vcpkg
with:
path: vcpkg
key: ${{ runner.os }}-vcpkg-${{ env.VCPKG_VERSION }}-${{ matrix.triplet }}

- name: Download vcpkg
if: steps.cache-vcpkg.outputs.cache-hit != 'true'
# wget doesn't seem to work under bash.
shell: 'powershell'
run: |
C:\msys64\usr\bin\wget.exe -nv `
https://github.com/microsoft/vcpkg/archive/refs/tags/${{ env.VCPKG_VERSION }}.zip `
-O vcpkg.zip
- name: Bootstrap vcpkg
if: steps.cache-vcpkg.outputs.cache-hit != 'true'
shell: 'bash'
run: |
set -x
unzip -q vcpkg.zip
rm -rf ${VCPKG_ROOT}
mv vcpkg-${VCPKG_VERSION} ${VCPKG_ROOT}
${VCPKG_ROOT}/bootstrap-vcpkg.sh
- name: Install libraries with vcpkg
shell: 'bash'
run: |
set -x
${VCPKG_ROOT}/vcpkg --triplet ${{ matrix.triplet }} install \
giflib \
libjpeg-turbo \
libpng \
libwebp \
#
- name: Configure
shell: 'bash'
run: |
set -x
mkdir build
cmake -Bbuild -H. ${{ matrix.arch }} \
-DBUILD_TESTING=OFF \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=`pwd`/prefix \
-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \
-DJPEGXL_ENABLE_OPENEXR=OFF \
-DJPEGXL_ENABLE_PLUGINS=OFF \
-DJPEGXL_ENABLE_TCMALLOC=OFF \
-DJPEGXL_ENABLE_VIEWERS=OFF \
-DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} \
#
- name: Build
shell: 'bash'
run: |
set -x
cmake --build build --config Release
- name: Install
shell: 'bash'
run: |
set -x
cmake --build build --config Release --target install
for pkg in giflib libjpeg-turbo libpng libwebp zlib; do
cp vcpkg/installed/${{matrix.triplet}}/share/${pkg}/copyright \
prefix/bin/LICENSE.${pkg}
done
cp third_party/sjpeg/COPYING prefix/bin/LICENSE.sjpeg
cp third_party/skcms/LICENSE prefix/bin/LICENSE.skcms
cp third_party/highway/LICENSE prefix/bin/LICENSE.highway
cp third_party/brotli/LICENSE prefix/bin/LICENSE.brotli
cp LICENSE prefix/bin/LICENSE.libjxl
- name: Upload artifacts
uses: actions/upload-artifact@v2
with:
name: jxl-${{matrix.triplet}}
path: |
prefix/bin/*
- name: Package release zip
if: github.event_name == 'release'
shell: 'powershell'
run: |
Compress-Archive -Path prefix\bin\* `
-DestinationPath ${{ runner.workspace }}\release_file.zip
- name: Upload binaries to release
if: github.event_name == 'release'
uses: svenstaro/upload-release-action@v1-release
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: ${{ runner.workspace }}/release_file.zip
asset_name: jxl-${{matrix.triplet}}.zip
tag: ${{ github.ref }}
overwrite: true
221 changes: 0 additions & 221 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
stages:
- build
- test
- release
- deploy

variables:
Expand Down Expand Up @@ -147,23 +146,6 @@ test:x86_64:clang:release:
# Quick run to make sure it doesn't crash. Not useful for actual benchmarks.
- ./ci.sh gbench --benchmark_min_time=0

# linux x86_64 with gcc in Debug mode.
build:x86_64:gcc7:debug:
<<: *linux_host_build_template
stage: build
script:
- CC=gcc-7 CXX=g++-7 SKIP_TEST=1 PACK_TEST=1 ./ci.sh debug
- tools/build_stats.py --save build/stats.json
cjxl djxl libjxl.so

test:x86_64:gcc7:debug:
<<: *linux_host_template
stage: test
dependencies:
- build:x86_64:gcc7:debug
script:
- TEST_STACK_LIMIT=1024 ./ci.sh test

# x86_84 test coverage build
build:x86_64:clang:coverage:
<<: *linux_host_build_template
Expand Down Expand Up @@ -219,21 +201,6 @@ pages:
- public


# x86_64 Scalar-only release runs only on master and on request.
build:x86_64:clang:scalar:
<<: *linux_host_build_template
stage: build
script:
- SKIP_TEST=1 PACK_TEST=1 CMAKE_CXX_FLAGS="-DHWY_DISABLED_TARGETS=~HWY_SCALAR" ./ci.sh release

test:x86_64:clang:scalar:
<<: *linux_host_template
stage: test
dependencies:
- build:x86_64:clang:scalar
script:
- ./ci.sh test

# i686 (x86 32-bit) builders
build:i686:clang:release:
<<: *linux_host_build_template
Expand Down Expand Up @@ -314,45 +281,6 @@ test:aarch64:clang:release-nhp:
# LeakSanitizer doesn't work when running under qemu-arm.
- ASAN_OPTIONS=detect_leaks=0 ./ci.sh test

# arm release
build:armhf:clang:release:
<<: *linux_host_build_template
stage: build
script:
- BUILD_TARGET=arm-linux-gnueabihf
CMAKE_CXX_FLAGS="-DJXL_DISABLE_SLOW_TESTS -march=armv7a -Wno-unused-command-line-argument" SKIP_TEST=1 PACK_TEST=1
STACK_SIZE=1
./ci.sh release -DJPEGXL_FORCE_NEON=ON
- tools/build_stats.py --save build/stats.json
--binutils=arm-linux-gnueabihf-
cjxl djxl libjxl.so

test:armhf:clang:release:
<<: *linux_host_template
stage: test
dependencies:
- build:armhf:clang:release
script:
- ./ci.sh test

# "asan", "tsan" and "msan" builds only run on master, tags and when requested
# from the web. Only supported in x86_64 for now.
build:x86_64:clang:asan:
<<: *linux_host_build_template
stage: build
script:
- SKIP_TEST=1 PACK_TEST=1 ./ci.sh asan

test:x86_64:clang:asan:
<<: *linux_host_template
stage: test
dependencies:
- build:x86_64:clang:asan
script:
- ./ci.sh test
# Quick run to make sure it doesn't crash. Not useful for actual benchmarks.
- ./ci.sh gbench --benchmark_min_time=0

# aarch64 asan build and test.
build:aarch64:clang:asan:
<<: *linux_host_build_template
Expand Down Expand Up @@ -386,52 +314,6 @@ test:x86_64:clang:tsan:
# tsan test runs fail with a small stack.
- TEST_STACK_LIMIT=1024 ./ci.sh test

build:x86_64:clang:msan:
<<: *linux_host_build_template
stage: build
script:
- SKIP_TEST=1 PACK_TEST=1 ./ci.sh msan

test:x86_64:clang:msan:
<<: *linux_host_template
stage: test
dependencies:
- build:x86_64:clang:msan
script:
- ./ci.sh test

# Merge request linter checks.
build:lint:
<<: *linux_host_template
stage: build
only:
- merge_requests
script:
- LINT_OUTPUT=clang-format.patch ./ci.sh lint
allow_failure: true
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
expire_in: 1 month
when: on_failure
paths:
- clang-format.patch

# Most clang-tidy checks are currently informational only since these are
# currently broken, however several are marked as errors.
build:tidy:
<<: *linux_host_build_template
stage: build
only:
- merge_requests
script:
- CMAKE_BUILD_TYPE="Release" ./ci.sh tidy
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
expire_in: 1 month
when: always
paths:
- build/clang-tidy.txt

# Faster benchmark over a smaller set of images.
.benchmark_x86_64_template: &benchmark_x86_64_template
<<: *linux_host_template
Expand All @@ -449,20 +331,6 @@ test:fast_benchmark:release:
script:
- STORE_IMAGES=0 ./ci.sh fast_benchmark

test:fast_benchmark:asan:
<<: *benchmark_x86_64_template
dependencies:
- build:x86_64:clang:asan
script:
- STORE_IMAGES=0 ./ci.sh fast_benchmark

test:fast_benchmark.msan:
<<: *benchmark_x86_64_template
dependencies:
- build:x86_64:clang:msan
script:
- STORE_IMAGES=0 ./ci.sh fast_benchmark

# This template runs on actual aarch64 hardware.
.benchmark_aarch64_template: &benchmark_aarch64_template
<<: *linux_host_template
Expand Down Expand Up @@ -564,92 +432,3 @@ test:ems_simd:all:
- export V8=/opt/.jsvu/v8
- source /opt/emsdk/emsdk_env.sh
- BUILD_TARGET=wasm32 emconfigure ./ci.sh test

### Release builds

# Debian package build template.
.linux_host_debian_template: &linux_host_debian_template
<<: *linux_host_template
variables:
GIT_SUBMODULE_STRATEGY: recursive
BUILD_DIR: build
# Prevent "apt" asking interactive questions.
DEBIAN_FRONTEND: noninteractive
TARGET_DIR: jpegxl-$CI_COMMIT_REF_NAME-$CI_BUILD_NAME
stage: release
script:
- apt update
- apt install -y devscripts build-essential
# Build and install highway.
- apt build-dep -y ./third_party/highway
- ./ci.sh debian_build highway
- dpkg -i build/debs/libhwy-dev_*_amd64.deb
# Build jpeg-xl
- apt build-dep -y .
- ./ci.sh debian_build jpeg-xl
# Print package stats.
- ./ci.sh debian_stats
- mkdir -p ${TARGET_DIR}
- mv -v build/debs/*.* ${TARGET_DIR}/
artifacts:
name: "jpegxl-$CI_COMMIT_REF_NAME-$CI_BUILD_NAME"
expire_in: 1 week
paths:
- jpegxl-*/*

# Debian package build.
debian10:
<<: *linux_host_debian_template
image: debian:10.6

ubuntu18.04:
<<: *linux_host_debian_template
before_script:
# In Ubuntu 18.04 no package installed the libgtest.a. libgtest-dev installs
# the source files only.
- apt update
- apt install -y build-essential libgtest-dev cmake
- for prj in googletest googlemock; do
(cd /usr/src/googletest/${prj}/ &&
cmake CMakeLists.txt -DCMAKE_INSTALL_PREFIX=/usr &&
make all install)
done
# Remove libgmock-dev dependency in Ubuntu 18.04. It doesn't exist there.
- sed '/libgmock-dev,/d' -i debian/control
image: ubuntu:18.04

ubuntu20.04:
<<: *linux_host_debian_template
image: ubuntu:20.04

# Static builds.
x86_64:static:
<<: *linux_host_build_template
stage: release
script:
# OpenEXR doesn't install the static libraries in the docker image.
- SKIP_TEST=1 ./ci.sh release
-DJPEGXL_DEP_LICENSE_DIR=/usr/share/doc
-DJPEGXL_STATIC=ON
-DBUILD_TESTING=OFF
-DJPEGXL_ENABLE_VIEWERS=OFF
-DJPEGXL_ENABLE_PLUGINS=OFF
-DJPEGXL_ENABLE_OPENEXR=OFF

win64:static:
<<: *linux_host_build_template
stage: release
script:
# OpenEXR doesn't install the static libraries in the docker image.
- BUILD_TARGET=x86_64-w64-mingw32 SKIP_TEST=1 ./ci.sh release
-DJPEGXL_DEP_LICENSE_DIR=/usr/share/doc
-DJPEGXL_STATIC=ON
-DBUILD_TESTING=OFF
-DJPEGXL_ENABLE_VIEWERS=OFF
-DJPEGXL_ENABLE_PLUGINS=OFF
-DJPEGXL_ENABLE_OPENEXR=OFF
-DJPEGXL_ENABLE_TCMALLOC=OFF
-DJPEGXL_ENABLE_FUZZERS=OFF
# Make sure the binary runs without any other environment variables set.
- wine64 build/tools/cjxl.exe
third_party/testdata/wesaturate/64px/cvo9xd_keong_macan_srgb8.png
9 changes: 0 additions & 9 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,6 @@
[submodule "third_party/skcms"]
path = third_party/skcms
url = https://skia.googlesource.com/skcms
[submodule "third_party/IQA-optimization"]
path = third_party/IQA-optimization
url = https://github.com/veluca93/IQA-optimization.git
[submodule "third_party/vmaf"]
path = third_party/vmaf
url = https://github.com/Netflix/vmaf.git
[submodule "third_party/difftest_ng"]
path = third_party/difftest_ng
url = https://github.com/thorfdbg/difftest_ng.git
[submodule "third_party/highway"]
path = third_party/highway
url = https://github.com/google/highway
7 changes: 5 additions & 2 deletions AUTHORS
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@ Google LLC <*@google.com>
Alexander Sago <cagelight@gmail.com>
Dirk Lemstra <dirk@lemstra.org>
Jon Sneyers <jon@cloudinary.com>
Pieter Wuille
Lovell Fuller
Marcin Konicki <ahwayakchih@gmail.com>
Ziemowit Zabawa <ziemek.zabawa@outlook.com>
Petr Diblík
Pieter Wuille
xiota
Ziemowit Zabawa <ziemek.zabawa@outlook.com>
Andrius Lukas Narbutas <andrius4669@gmail.com>
52 changes: 51 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,60 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
## [0.6] - 2021-10-04
### Added
- API: New functions to decode extra channels:
`JxlDecoderExtraChannelBufferSize` and `JxlDecoderSetExtraChannelBuffer`.
- API: New function `JxlEncoderInitBasicInfo` to initialize `JxlBasicInfo`
(only needed when encoding). NOTE: it is now required to call this function
when using the encoder. Padding was added to the struct for forward
compatibility.
- API: Support for encoding oriented images.
- API: FLOAT16 support in the encoder API.
- Rewrite of the GDK pixbuf loader plugin. Added proper color management and
animation support.
- Rewrite of GIMP plugin. Added compression parameters dialog and switched to
using the public C API.
- Debian packages for GDK pixbuf loader (`libjxl-gdk-pixbuf`) and GIMP
(`libjxl-gimp-plugin`) plugins.
- `cjxl`/`djxl` support for `stdin` and `stdout`.

### Changed
- API: Renamed the field `alpha_associated` in `JxlExtraChannelInfo` to
`alpha_premultiplied`, to match the corresponding name in `JxlBasicInfo`.
- Improved the 2x2 downscaling method in the encoder for the optional color
channel resampling for low bit rates.
- Fixed: the combination of floating point original data, XYB color encoding,
and Modular mode was broken (in both encoder and decoder). It now works.
NOTE: this can cause the current encoder to write jxl bitstreams that do
not decode with the old decoder. In particular this will happen when using
cjxl with PFM, EXR, or floating point PSD input, and a combination of XYB
and modular mode is used (which caused an encoder error before), e.g.
using options like `-m -q 80` (lossy modular), `-d 4.5` or `--progressive_dc=1`
(modular DC frame), or default lossy encoding on an image where patches
end up being used. There is no problem when using cjxl with PNG, JPEG, GIF,
APNG, PPM, PGM, PGX, or integer (8-bit or 16-bit) PSD input.
- `libjxl` static library now bundles skcms, fixing static linking in
downstream projects when skcms is used.
- Spline rendering performance improvements.
- Butteraugli changes for less visual masking.

## [0.5] - 2021-08-02
### Added
- API: New function to decode the image using a callback outputting a part of a
row per call.
- API: 16-bit float output support.
- API: `JxlDecoderRewind` and `JxlDecoderSkipFrames` functions to skip more
efficiently to earlier animation frames.
- API: `JxlDecoderSetPreferredColorProfile` function to choose color profile in
certain circumstances.
- encoder: Adding `center_x` and `center_y` flags for more control of the tile
order.
- New encoder speeds `lightning` (1) and `thunder` (2).

### Changed
- Re-licensed the project under a BSD 3-Clause license. See the
[LICENSE](LICENSE) and [PATENTS](PATENTS) files for details.
- Full JPEG XL part 1 specification support: Implemented all the spec required
to decode files to pixels, including cases that are not used by the encoder
yet. Part 2 of the spec (container format) is final but not fully implemented
Expand All @@ -22,6 +69,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Reduce the size of the jxl_dec library by removing dependencies.
- A few encoding speedups.
- Clarify the security policy.
- Significant encoding improvements (~5 %) and less ringing.
- Butteraugli metric to have some less masking.
- `cjxl` flag `--speed` is deprecated and replaced by the `--effort` synonym.

### Removed
- API for returning a downsampled DC was deprecated
Expand Down
83 changes: 66 additions & 17 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -75,18 +75,24 @@ set(JPEGXL_ENABLE_FUZZERS ${ENABLE_FUZZERS_DEFAULT} CACHE BOOL
"Build JPEGXL fuzzer targets.")
set(JPEGXL_ENABLE_DEVTOOLS false CACHE BOOL
"Build JPEGXL developer tools.")
set(JPEGXL_ENABLE_TOOLS true CACHE BOOL
"Build JPEGXL user tools: cjxl and djxl.")
set(JPEGXL_ENABLE_MANPAGES true CACHE BOOL
"Build and install man pages for the command-line tools.")
set(JPEGXL_ENABLE_BENCHMARK true CACHE BOOL
"Build JPEGXL benchmark tools.")
set(JPEGXL_ENABLE_EXAMPLES true CACHE BOOL
"Build JPEGXL library usage examples.")
set(JPEGXL_ENABLE_JNI true CACHE BOOL
"Build JPEGXL JNI Java wrapper, if Java dependencies are installed.")
set(JPEGXL_ENABLE_SJPEG true CACHE BOOL
"Build JPEGXL with support for encoding with sjpeg.")
set(JPEGXL_ENABLE_OPENEXR true CACHE BOOL
"Build JPEGXL with support for OpenEXR if available.")
set(JPEGXL_ENABLE_SKCMS true CACHE BOOL
"Build with skcms instead of lcms2.")
set(JPEGXL_BUNDLE_SKCMS true CACHE BOOL
"When building with skcms, bundle it into libjxl.a.")
set(JPEGXL_ENABLE_VIEWERS false CACHE BOOL
"Build JPEGXL viewer tools for evaluation.")
set(JPEGXL_ENABLE_TCMALLOC ${ENABLE_TCMALLOC_DEFAULT} CACHE BOOL
Expand Down Expand Up @@ -150,8 +156,20 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
if(JPEGXL_STATIC)
set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
set(BUILD_SHARED_LIBS 0)
set(CMAKE_EXE_LINKER_FLAGS
"${CMAKE_EXE_LINKER_FLAGS} -static -static-libgcc -static-libstdc++")
# Clang developers say that in case to use "static" we have to build stdlib
# ourselves; for real use case we don't care about stdlib, as it is "granted",
# so just linking all other libraries is fine.
if (NOT APPLE)
set(CMAKE_EXE_LINKER_FLAGS
"${CMAKE_EXE_LINKER_FLAGS} -static -static-libgcc -static-libstdc++")
endif()
endif() # JPEGXL_STATIC

# Threads
set(THREADS_PREFER_PTHREAD_FLAG YES)
find_package(Threads REQUIRED)

if(JPEGXL_STATIC)
if (MINGW)
# In MINGW libstdc++ uses pthreads directly. When building statically a
# program (regardless of whether the source code uses pthread or not) the
Expand All @@ -167,9 +185,22 @@ if(JPEGXL_STATIC)
# will be discarded anyway.
# This adds these flags as dependencies for *all* targets. Adding this to
# CMAKE_EXE_LINKER_FLAGS instead would cause them to be included before any
# object files and therefore discarded.
# object files and therefore discarded. This should be set in the
# INTERFACE_LINK_LIBRARIES of Threads::Threads but some third_part targets
# don't depend on it.
link_libraries(-Wl,-Bstatic -lstdc++ -lpthread -Wl,-Bdynamic)
endif() # MINGW
elseif(CMAKE_USE_PTHREADS_INIT)
# "whole-archive" is not supported on OSX.
if (NOT APPLE)
# Set pthreads as a whole-archive, otherwise weak symbols in the static
# libraries will discard pthreads symbols leading to segmentation fault at
# runtime.
message(STATUS "Using -lpthread as --whole-archive")
set_target_properties(Threads::Threads PROPERTIES
INTERFACE_LINK_LIBRARIES
"-Wl,--whole-archive;-lpthread;-Wl,--no-whole-archive")
endif()
endif()
endif() # JPEGXL_STATIC

if (MSVC)
Expand Down Expand Up @@ -249,9 +280,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED YES)

add_subdirectory(third_party)

set(THREADS_PREFER_PTHREAD_FLAG YES)
find_package(Threads REQUIRED)

# Copy the JXL license file to the output build directory.
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/LICENSE"
${PROJECT_BINARY_DIR}/LICENSE.jpeg-xl COPYONLY)
Expand Down Expand Up @@ -294,13 +322,31 @@ add_custom_target(doc false
endif() # DOXYGEN_FOUND

if(JPEGXL_ENABLE_MANPAGES)
find_package(Python COMPONENTS Interpreter)
if(Python_Interpreter_FOUND)
find_program(ASCIIDOC a2x)
endif()
if(NOT Python_Interpreter_FOUND OR "${ASCIIDOC}" STREQUAL "ASCIIDOC-NOTFOUND")
message(WARNING "asciidoc was not found, the man pages will not be installed.")
find_program(ASCIIDOC a2x)
if(NOT "${ASCIIDOC}" STREQUAL "ASCIIDOC-NOTFOUND")
file(STRINGS "${ASCIIDOC}" ASCIIDOC_SHEBANG LIMIT_COUNT 1)
if(ASCIIDOC_SHEBANG MATCHES "python2")
find_package(Python2 COMPONENTS Interpreter)
set(ASCIIDOC_PY_FOUND "${Python2_Interpreter_FOUND}")
set(ASCIIDOC_PY Python2::Interpreter)
elseif(ASCIIDOC_SHEBANG MATCHES "python3")
find_package(Python3 COMPONENTS Interpreter)
set(ASCIIDOC_PY_FOUND "${Python3_Interpreter_FOUND}")
set(ASCIIDOC_PY Python3::Interpreter)
else()
find_package(Python COMPONENTS Interpreter QUIET)
if(NOT Python_Interpreter_FOUND)
find_program(ASCIIDOC_PY python)
if(NOT ASCIIDOC_PY STREQUAL "ASCIIDOC_PY-NOTFOUND")
set(ASCIIDOC_PY_FOUND ON)
endif()
else()
set(ASCIIDOC_PY_FOUND "${Python_Interpreter_FOUND}")
set(ASCIIDOC_PY Python::Interpreter)
endif()
endif()

if (ASCIIDOC_PY_FOUND)
set(MANPAGE_FILES "")
set(MANPAGES "")
foreach(PAGE IN ITEMS cjxl djxl)
Expand All @@ -309,7 +355,7 @@ else()
# does not recognize it.
add_custom_command(
OUTPUT "${PAGE}.1"
COMMAND Python::Interpreter
COMMAND "${ASCIIDOC_PY}"
ARGS "${ASCIIDOC}"
--format manpage --destination-dir="${CMAKE_CURRENT_BINARY_DIR}"
"${CMAKE_CURRENT_SOURCE_DIR}/doc/man/${PAGE}.txt"
Expand All @@ -319,12 +365,15 @@ else()
endforeach()
add_custom_target(manpages ALL DEPENDS ${MANPAGES})
install(FILES ${MANPAGE_FILES} DESTINATION share/man/man1)
endif()
endif()
endif() # ASCIIDOC_PY_FOUND
else()
message(WARNING "asciidoc was not found, the man pages will not be installed.")
endif() # ASCIIDOC != "ASCIIDOC-NOTFOUND"
endif() # JPEGXL_ENABLE_MANPAGES

# Example usage code.
if (${JPEGXL_ENABLE_EXAMPLES})
add_subdirectory(examples)
include(examples/examples.cmake)
endif ()

# Plugins for third-party software
Expand Down
43 changes: 25 additions & 18 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@
<img src="doc/jxl.svg" width="100" align="right" alt="JXL logo">

This repository contains a reference implementation of JPEG XL (encoder and
decoder), called `libjxl`.
decoder), called `libjxl`. This software library is
[used by many applications that support JPEG XL](doc/software_support.md).

JPEG XL is in the final stages of standardization and its codestream format is
frozen.
JPEG XL is in the final stages of standardization and its codestream and file format
are frozen.

The libraries API, command line options and tools in this repository are subject
The library API, command line options, and tools in this repository are subject
to change, however files encoded with `cjxl` conform to the JPEG XL format
specification and can be decoded with current and future `djxl` decoders or
`libjxl` decoding library.
Expand All @@ -24,9 +25,12 @@ git clone https://github.com/libjxl/libjxl.git --recursive
```

This repository uses git submodules to handle some third party dependencies
under `third_party/`, that's why is important to pass `--recursive`. If you
under `third_party`, that's why is important to pass `--recursive`. If you
didn't check out with `--recursive`, or any submodule has changed, run:
`git submodule update --init --recursive`.

```bash
git submodule update --init --recursive
```

Important: If you downloaded a zip file or tarball from the web interface you
won't get the needed submodules and the code will not compile. You can download
Expand All @@ -50,11 +54,11 @@ sudo apt install libgif-dev libjpeg-dev libopenexr-dev libpng-dev libwebp-dev
```

We recommend using a recent Clang compiler (version 7 or newer), for that
install clang and set `CC` and `CXX` variables. For example, with clang-7:
install clang and set `CC` and `CXX` variables.

```bash
sudo apt install clang-7
export CC=clang-7 CXX=clang++-7
sudo apt install clang
export CC=clang CXX=clang++
```

### Building
Expand Down Expand Up @@ -112,15 +116,19 @@ For more comprehensive benchmarking options, see the
We build a common environment based on Debian/Ubuntu using Docker. Other
systems may have different combinations of versions and dependencies that
have not been tested and may not work. For those cases we recommend using the
Docker environment as explained in the
Docker container as explained in the
[step by step guide](doc/developing_in_docker.md).

### Building JPEG XL for developers

For experienced developers, we also provide build instructions for an [up to
date Debian-based Linux](doc/developing_in_debian.md) and [64-bit
Windows](doc/developing_in_windows.md). If you encounter any difficulties,
please use Docker instead.
For experienced developers, we provide build instructions for several other environments:

* [Building on Debian](doc/developing_in_debian.md)
* Building on Windows with [vcpkg](doc/developing_in_windows_vcpkg.md) (Visual Studio 2019)
* Building on Windows with [MSYS2](doc/developing_in_windows_msys.md)
* [Cross Compiling for Windows with Crossroad](doc/developing_with_crossroad.md)

If you encounter any difficulties, please use Docker instead.

## License

Expand All @@ -144,11 +152,10 @@ format: Cloudinary and Google.
* [JPEG XL community website](https://jpegxl.info)

### Development process
* [Docker setup - **start here**](doc/developing_in_docker.md)
* [Building on Debian](doc/developing_in_debian.md) - for experts only
* [Building on Windows](doc/developing_in_windows.md) - for experts only

* [More information on testing/build options](doc/building_and_testing.md)
* [Git guide for JPEG XL](doc/developing_in_github.md) - for developers only
* [Git guide for JPEG XL](doc/developing_in_github.md) - for developers
* [Fuzzing](doc/fuzzing.md) - for developers
* [Building Web Assembly artifacts](doc/building_wasm.md)

### Contact
Expand Down
15 changes: 15 additions & 0 deletions bash_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,21 @@ EOF
return $ret
}

# Test that we don't use %n in C++ code to avoid using it in printf and scanf.
# This test is not very precise but in cases where "module n" is needed we would
# normally have "% n" instead of "%n". Using %n is not allowed in Android 10+.
test_percent_n() {
local ret=0
local f
for f in $(git ls-files | grep -E '(\.cc|\.cpp|\.h)$'); do
if grep -i -H -n -E '%h*n' "$f" >&2; then
echo "Don't use \"%n\"." >&2
ret=1
fi
done
return ${ret}
}

main() {
local ret=0
cd "${MYDIR}"
Expand Down
58 changes: 56 additions & 2 deletions ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ if [[ ! -z "${HWY_BASELINE_TARGETS}" ]]; then
fi

# Version inferred from the CI variables.
CI_COMMIT_SHA=${CI_COMMIT_SHA:-}
CI_COMMIT_SHA=${CI_COMMIT_SHA:-${GITHUB_SHA:-}}
JPEGXL_VERSION=${JPEGXL_VERSION:-${CI_COMMIT_SHA:0:8}}

# Benchmark parameters
Expand Down Expand Up @@ -1187,7 +1187,7 @@ cmd_fuzz() {
cmd_lint() {
merge_request_commits
{ set +x; } 2>/dev/null
local versions=(${1:-6.0 7 8 9})
local versions=(${1:-6.0 7 8 9 10 11})
local clang_format_bins=("${versions[@]/#/clang-format-}" clang-format)
local tmpdir=$(mktemp -d)
CLEANUP_FILES+=("${tmpdir}")
Expand Down Expand Up @@ -1360,6 +1360,60 @@ cmd_debian_build() {
esac
}

get_version() {
local varname=$1
local line=$(grep -F "set(${varname} " lib/CMakeLists.txt | head -n 1)
[[ -n "${line}" ]]
line="${line#set(${varname} }"
line="${line%)}"
echo "${line}"
}

cmd_bump_version() {
local newver="${1:-}"

if ! which dch >/dev/null; then
echo "Run:\n sudo apt install debhelper"
exit 1
fi

if [[ -z "${newver}" ]]; then
local major=$(get_version JPEGXL_MAJOR_VERSION)
local minor=$(get_version JPEGXL_MINOR_VERSION)
local patch=0
minor=$(( ${minor} + 1))
else
local major="${newver%%.*}"
newver="${newver#*.}"
local minor="${newver%%.*}"
newver="${newver#${minor}}"
local patch="${newver#.}"
if [[ -z "${patch}" ]]; then
patch=0
fi
fi

newver="${major}.${minor}"
if [[ "${patch}" != "0" ]]; then
newver="${newver}.${patch}"
fi
echo "Bumping version to ${newver} (${major}.${minor}.${patch})"
sed -E \
-e "s/(set\\(JPEGXL_MAJOR_VERSION) [0-9]+\\)/\\1 ${major})/" \
-e "s/(set\\(JPEGXL_MINOR_VERSION) [0-9]+\\)/\\1 ${minor})/" \
-e "s/(set\\(JPEGXL_PATCH_VERSION) [0-9]+\\)/\\1 ${patch})/" \
-i lib/CMakeLists.txt

# Update lib.gni
tools/build_cleaner.py --update

# Mark the previous version as "unstable".
DEBCHANGE_RELEASE_HEURISTIC=log dch -M --distribution unstable --release ''
DEBCHANGE_RELEASE_HEURISTIC=log dch -M \
--newversion "${newver}" \
"Bump JPEG XL version to ${newver}."
}

# Check that the AUTHORS file contains the email of the committer.
cmd_authors() {
merge_request_commits
Expand Down
12 changes: 12 additions & 0 deletions debian/changelog
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
jpeg-xl (0.6) UNRELEASED; urgency=medium

* Bump JPEG XL version to 0.6.

-- JPEG XL Maintainers <jpegxl@google.com> Thu, 12 Aug 2021 23:49:40 +0200

jpeg-xl (0.5.0) unstable; urgency=medium

* Bump JPEG XL version to 0.5.0.

-- JPEG XL Maintainers <jpegxl@google.com> Thu, 12 Aug 2021 23:49:40 +0200

jpeg-xl (0.3.7) UNRELEASED; urgency=medium

* Bump JPEG XL version to 0.3.7.
Expand Down
55 changes: 41 additions & 14 deletions debian/control
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,26 @@ Maintainer: JPEG XL Maintainers <jpegxl@google.com>
Section: misc
Priority: optional
Standards-Version: 3.9.8
Build-Depends: cmake,
debhelper (>= 9),
libbrotli-dev,
libgif-dev,
libgmock-dev,
libgoogle-perftools-dev,
libgtest-dev,
libhwy-dev,
libjpeg-dev,
libopenexr-dev,
libpng-dev,
libwebp-dev,
pkg-config,
Homepage: https://gitlab.com/wg1/jpeg-xl
Build-Depends:
asciidoc,
cmake,
debhelper (>= 9),
libbrotli-dev,
libgdk-pixbuf-2.0-dev | libgdk-pixbuf2.0-dev,
libgif-dev,
libgimp2.0-dev,
libgmock-dev,
libgoogle-perftools-dev,
libgtest-dev,
libhwy-dev,
libjpeg-dev,
libopenexr-dev,
libpng-dev,
libwebp-dev,
pkg-config,
xdg-utils,
xmlto,
Homepage: https://github.com/libjxl/libjxl
Rules-Requires-Root: no

Package: jxl
Expand All @@ -36,6 +42,7 @@ Package: libjxl-dev
Architecture: any
Section: libdevel
Depends: libjxl (= ${binary:Version}), ${misc:Depends}
libhwy-dev,
Description: JPEG XL Image Coding System - "JXL" (development files)
The JPEG XL Image Coding System (ISO/IEC 18181) is a lossy and
lossless image compression format. It has a rich feature set and is
Expand All @@ -59,3 +66,23 @@ Description: JPEG XL Image Coding System - "JXL" (shared libraries)
several features that help transition from the legacy JPEG format.
.
This package installs shared libraries.

Package: libjxl-gdk-pixbuf
Architecture: any
Multi-Arch: same
Section: libs
Depends: ${shlibs:Depends}, ${misc:Depends}
Pre-Depends: ${misc:Pre-Depends}
Description: JPEG XL Plugin for gdk-pixbuf
This package installs the required files for reading JPEG XL files in
GTK applications.

Package: libjxl-gimp-plugin
Architecture: any
Multi-Arch: same
Section: graphics
Depends: ${shlibs:Depends}, ${misc:Depends}
Pre-Depends: ${misc:Pre-Depends}
Enhances: gimp
Description: JPEG XL Import and Export Plugin for GIMP
This is a plugin for GIMP version 2.10.x to import and export JPEG XL images.
4 changes: 3 additions & 1 deletion debian/jxl.install
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
debian/tmp/usr/bin/*
usr/bin/*
usr/share/man/man1/cjxl.1
usr/share/man/man1/djxl.1
8 changes: 4 additions & 4 deletions debian/libjxl-dev.install
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
debian/tmp/usr/include/jxl/*.h
debian/tmp/usr/lib/*/*.a
debian/tmp/usr/lib/*/*.so
debian/tmp/usr/lib/*/pkgconfig/*.pc
usr/include/jxl/*.h
usr/lib/*/*.a
usr/lib/*/*.so
usr/lib/*/pkgconfig/*.pc
3 changes: 3 additions & 0 deletions debian/libjxl-gdk-pixbuf.install
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
usr/lib/*/gdk-pixbuf-*/*/loaders/*
usr/share/mime/packages/image-jxl.xml
usr/share/thumbnailers/jxl.thumbnailer
1 change: 1 addition & 0 deletions debian/libjxl-gimp-plugin.install
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
usr/lib/gimp
2 changes: 1 addition & 1 deletion debian/libjxl.install
Original file line number Diff line number Diff line change
@@ -1 +1 @@
debian/tmp/usr/lib/*/libjxl*.so.*
usr/lib/*/libjxl*.so.*
7 changes: 6 additions & 1 deletion debian/rules
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
#!/usr/bin/make -f

include /usr/share/dpkg/pkg-info.mk

%:
dh $@ --buildsystem=cmake

override_dh_auto_configure:
# TODO(deymo): Remove the DCMAKE_BUILD_TYPE once builds without NDEBUG
# are as useful as Release builds.
dh_auto_configure -- \
-DJPEGXL_VERSION=$(DEB_VERSION) \
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DJPEGXL_FORCE_SYSTEM_GTEST=ON \
-DJPEGXL_FORCE_SYSTEM_BROTLI=ON \
-DJPEGXL_FORCE_SYSTEM_HWY=ON
-DJPEGXL_FORCE_SYSTEM_HWY=ON \
-DJPEGXL_ENABLE_PLUGINS=ON
2 changes: 1 addition & 1 deletion doc/building_and_testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

This file describes the building and testing facilities provided by the `ci.sh`
script. It assumes you already have the build environment set up, preferably
Docker (see [instructions](building_in_docker.md)).
Docker (see [instructions](developing_in_docker.md)).

## Basic building

Expand Down
2 changes: 1 addition & 1 deletion doc/building_wasm.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ variables are set:
## Requirements

[CMake](https://cmake.org/) is used as a build system. To install it, follow
[Debian build instructions](building_in_debian.md).
[Debian build instructions](developing_in_debian.md).

[Emscripten SDK](https://emscripten.org/) is required for building
WebAssembly artifacts. To install it, follow the
Expand Down
19 changes: 11 additions & 8 deletions doc/developing_in_debian.md
Original file line number Diff line number Diff line change
@@ -1,26 +1,29 @@
# Developing in Debian

These instructions assume an up-to-date Debian/Ubuntu system.
For other platforms, please instead use the [Docker container](developing_in_docker.md).
For other platforms, please instead use the following:

* [Developing in Docker](developing_in_docker.md).
* [Cross Compiling for Windows with Crossroad](developing_with_crossroad.md).

## Minimum build dependencies

Apart from the dependencies in third_party, some of the tools use external
dependencies that need to be installed in your system first:
Apart from the dependencies in `third_party`, some of the tools use external
dependencies that need to be installed on your system first:

```bash
sudo apt install cmake clang doxygen g++-8 extra-cmake-modules libgif-dev \
sudo apt install cmake clang doxygen g++ extra-cmake-modules libgif-dev \
libjpeg-dev ninja-build libgoogle-perftools-dev
```

Make sure your default "clang" compiler is at least version 6 running
Make sure your default `clang` compiler is at least version 6 by running

```bash
clang --version
```

If it still shows an old version despite having for example a clang-7 installed, you need
to update the default clang compiler. In Debian-based systems run:
If it still shows an old version despite having, for example, `clang-7` installed, you need
to update the default `clang` compiler. On Debian-based systems run:

```bash
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-7 100
Expand All @@ -43,7 +46,7 @@ sudo apt install clang-format clang-tidy curl parallel gcovr

## Building

The project uses CMake to build. We provide a script that simplifies the
The `libjxl` project uses CMake to build. We provide a script that simplifies the
invocation. To build and test the project, run

```bash
Expand Down
88 changes: 42 additions & 46 deletions doc/developing_in_docker.md
Original file line number Diff line number Diff line change
@@ -1,20 +1,19 @@
# Developing in docker
# Developing in Docker

Docker allows to run software in a packaged container, isolated from your host
system. This is useful to run the code in a standard environment instead of
dealing with problems caused by different building environments during
development, as well as to simplify external dependencies by including them
in the automated setup of this standard environment.
Docker allows software to be run in a packaged container, isolated from the
host system. This allows code to be run in a standard environment instead
of dealing with different build environments during development. It also
simplifies resolving external dependencies by including them in the automated
setup of the container environment.

## Set up the container

Read over the [docker install instructions](https://docs.docker.com/install/) to
get docker installed on your computer. As of 2020-01, this requires creating a
free account with Docker.
You can read installation instructions and download Docker for your
operating system at [Get Docker](https://docs.docker.com/get-docker/).

The image used by our builders is an ubuntu-bionic image with all the required
dependencies and build tools installed. You can pull this image from
`gcr.io/jpegxl/jpegxl-builder` using the following command:
The image used by our builders is an Ubuntu Bionic image with all the
required dependencies and build tools installed. You can pull this image
from `gcr.io/jpegxl/jpegxl-builder` using the following command:

```bash
sudo docker pull gcr.io/jpegxl/jpegxl-builder
Expand All @@ -29,20 +28,21 @@ sudo docker run -it --rm \
gcr.io/jpegxl/jpegxl-builder bash
```

This creates and runs a container that will be deleted after you exit from this
This creates and runs a container that will be deleted after you exit the
terminal (`--rm` flag).

The `-v` flag is to map the directory containing your jpeg-xl checkout in your
host (assumed to be at `$HOME/jpeg-xl`) to a directory inside the container at
/jpeg-xl. This means that whenever you make changes to the code from your host
using your favorite editor they are also changed in the container, since the
directory is simply mounted into the container.
/jpeg-xl. Since the container is accessing the host folder directly,
changes made on the host will will be seen immediately in the container,
and vice versa.

On OSX, the path must be one of those whitelisted/shared with Docker. $HOME
On OSX, the path must be one of those shared and whitelisted with Docker. $HOME
(which is a subdirectory of /Users/) is known to work with the factory-default
settings of Docker.

On OSX, "cannot find name for group ID" can be ignored.
On OSX, you may ignore the warning that Docker "cannot find name for group ID".
This warning may also appear on some Linux computers.

On Windows, you can run the following from the jpeg-xl directory obtained from
Gitlab:
Expand All @@ -54,8 +54,9 @@ docker run -u root:root -it --rm -v %cd%:/jpeg-xl -w /jpeg-xl \

## Basic building

Inside the Docker container, you can compile everything and run unit tests
by running the following:
Inside the Docker container, you can compile everything and run unit tests.
We need to specify `clang-7` because the default `clang` compiler is
not installed on the image.

```bash
CC=clang-7 CXX=clang++-7 ./ci.sh opt
Expand All @@ -65,33 +66,28 @@ This writes binaries to `/jpeg-xl/build/tools` and runs unit tests.
More information on [build modes and testing](building_and_testing.md) is
available.

If there already was a build directory on the host this can give conflicts,
remove it first with `rm -rf build`.

Note that the default "clang" compiler is not installed on the image, hence we
specify clang-7. If a build/ directory already exists and was configured for
a different compiler, cmake will complain. This can be avoided by renaming any
existing build/ directory or setting the `BUILD_DIR` environment variable.
If a `build` directory already exists and was configured for a different
compiler, `cmake` will complain. This can be avoided by renaming or removing
the existing `build` directory or setting the `BUILD_DIR` environment variable.

## Cross-compiling environments (optional)

We have installed the required cross-compiling tools in the main docker image
`jpegxl-builder`. This allows to compile for other architectures such as arm
and run the tests emulated under qemu.

The docker container already has several `qemu-*-static` binaries (such as
`qemu-aarch64-static`) that emulate other architectures on x86_64. These qemu
binaries are automatically used when running a foreign architecture program in
the container only if you have `binfmt` installed and configured to use the
binaries from `/usr/bin/qemu-*-static` in the *host*. This is the default
location in Ubuntu/Debian, however you need to install on the host both
`binfmt-support` and `qemu-user-static` even if the qemu-user-static binaries
are not used from the host, since binfmt-support in Debian/Ubuntu only
configures the `binfmt` signatures of the architectures you have a
qemu-user-static binary installed on the host. If you have these configured
somewhere else in other distribution, symlink that location to the
`/usr/bin/qemu-*-static` inside the docker before running. To install binfmt
support in your Ubuntu host run *outside* the container:
We have installed the required cross-compiling tools in the main Docker image
`jpegxl-builder`. This allows compiling for other architectures, such as arm.
Tests will be emulated under `qemu`.

The Docker container has several `qemu-*-static` binaries (such as
`qemu-aarch64-static`) that emulate other architectures on x86_64. These
binaries are automatically used when running foreign architecture programs
in the container only if `binfmt` is installed and configured on the *host*
to use binaries from `/usr/bin` . This is the default location on Ubuntu/Debian.

You need to install both `binfmt-support` and `qemu-user-static` on the host,
since `binfmt-support` configures only `binfmt` signatures of architectures
that are installed. If these are configured elsewhere on other distributions,
you can symlink them to `/usr/bin/qemu-*-static` inside the Docker container.

To install binfmt support in your Ubuntu host run *outside* the container:

```bash
sudo apt install binfmt-support qemu-user-static
Expand All @@ -107,8 +103,8 @@ export BUILD_TARGET=aarch64-linux-gnu CC=clang-7 CXX=clang++-7
The `BUILD_TARGET=aarch64-linux-gnu` environment variable tells the `ci.sh`
script to cross-compile for that target. This also changes the default
`BUILD_DIR` to `build-aarch64` since you never want to mix them with the `build`
of your host. You can also explicitly set a `BUILD_DIR` environment variable and
that will be used instead. The list of supported BUILD_TARGET values for this
of your host. You can also explicitly set a `BUILD_DIR` environment variable
that will be used instead. The list of supported `BUILD_TARGET` values for this
container is:

* *the empty string* (for native x86_64 support)
Expand Down
2 changes: 1 addition & 1 deletion doc/developing_in_github.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ propose to include changes in the main repository via a Pull Request.

Once you are done you should have your repository at

https://github.com/{{USERNAME}}/libjxl
https://<!-- not a link -->github.com<!-- not a link -->/*{{USERNAME}}*/libjxl

where {{USERNAME}} denotes your GitHub username.

Expand Down
168 changes: 168 additions & 0 deletions doc/developing_in_windows_msys.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
# Developing for Windows with MSYS2

[MSYS2](https://www.msys2.org/) ("minimal system 2") is a software distribution and a development platform based on MinGW and Cygwin. It provides a Unix-like environment to build code on Windows. These instructions were written with a 64-bit instance of Windows 10 running on a VM. They may also work on native instances of Windows and other versions of Windows.

## Build Environments

MSYS2 provides multiple development [environments](https://www.msys2.org/docs/environments/). By convention, they are referred to in uppercase. They target slightly different platforms, runtime libraries, and compiler toolchains. For example, to build for 32-bit Windows, use the MINGW32 environment. For interoperability with Visual Studio projects, use the UCRT64 environment.

Since all of the build environments are built on top of the MSYS environment, **all updates and package installation must be done from within the MSYS environment**. After making any package changes, `exit` all MSYS2 terminals and restart the desired build-environment. This reminder is repeated multiple times throughout this guide.

* **MINGW32:** To compile for 32-bit Windows (on 64-bit Windows), use packages from the `mingw32` group. Package names are prefixed with `mingw-w64-i686`. The naming scheme may be different on the 32-bit version of MSYS2.

* **MINGW64:** This is the primary environment to building for 64-bit Windows. It uses the older MSVCRT runtime, which is widely available across Windows systems. Package names are prefixed with `mingw-w64-x86_64`.

* **UCRT64:** The Universal C Runtime (UCRT) is used by recent versions of Microsoft Visual Studio. It ships by default with Windows 10. For older versions of Windows, it must be provided with the application or installed by the user. Package names are prefixed with `mingw-w64-ucrt-x86_64`.

* **CLANG64:** Unfortunately, the `gimp` packages are not available for the CLANG64 environment. However, `libjxl` will otherwise build in this environment if the appropriate packages are installed. Packages are prefixed with `mingw-w64-clang-x86_64`.

## Install and Upgrade MSYS2

Download MSYS2 from the homepage. Install at a location without any spaces on a drive with ample free space. After installing the packages used in this guide, MSYS2 used about 15GB of space.

Toward the end of installation, select the option to run MSYS2 now. A command-line window will open. Run the following command, and answer the prompts to update the repository and close the terminal.

```bash
pacman -Syu
```

Now restart the MSYS environment and run the following command to complete updates:

```bash
pacman -Su
```

## Package Management

Packages are organized in groups, which share the build environment name, but in lower case. Then they have name prefixes that indicate which group they belong to. Consider this package search: `pacman -Ss cmake`

```
mingw32/mingw-w64-i686-cmake
mingw64/mingw-w64-x86_64-cmake
ucrt64/mingw-w64-ucrt-x86_64-cmake
clang64/mingw-w64-clang-x86_64-cmake
msys/cmake
```

We can see the organization `group/prefix-name`. When installing packages, the group name is optional.

```bash
pacman -S mingw-w64-x86_64-cmake
```

For tools that need to be aware of the compiler to function, install the package that corresponds with the specific build-environment you plan to use. For `cmake`, install the `mingw64` version. The generic `msys/cmake` will not function correctly because it will not find the compiler. For other tools, the generic `msys` version is adequate, like `msys/git`.

To remove packages, use:

```bash
pacman -Rsc [package-name]
```

## Worst-Case Scenario...

If packages management is done within a build environment other than MSYS, the environment structure will be disrupted and compilation will likely fail. If this happens, it may be necessary to reinstall MSYS2.

1. Rename the `msys64` folder to `msys64.bak`.

2. Use the installer to reinstall MSYS2 to `msys64`.

3. Copy packages from `msys64.bak/var/cache/pacman/pkg/` to the new installation to save download time and bandwidth.

4. Use `pacman` from within the MSYS environment to install and update packages.

5. After successfully building a project, it is safe to delete `msys64.bak`

## The MING64 Environment

Next set up the MING64 environment. The following commands should be run within the MSYS environment. `pacman -S` is used to install packages. The `--needed` argument prevents packages from being reinstalled.

```bash
pacman -S --needed base-devel mingw-w64-x86_64-toolchain
pacman -S git mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja \
mingw-w64-x86_64-gtest mingw-w64-x86_64-giflib \
mingw-w64-x86_64-libpng mingw-w64-x86_64-libjpeg-turbo
```

## Build `libjxl`

Download the source from the libjxl [releases](https://github.com/libjxl/libjxl/releases) page. Alternatively, you may obtain the latest development version with `git`. Run `./deps.sh` to ensure additional third-party dependencies are downloaded.

Start the MINGW64 environment, create a build directory within the source directory, and configure with `cmake`.

```bash
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DBUILD_TESTING=OFF -DJPEGXL_ENABLE_BENCHMARK=OFF \
-DJPEGXL_ENABLE_PLUGINS=ON -DJPEGXL_ENABLE_MANPAGES=OFF \
-DJPEGXL_FORCE_SYSTEM_BROTLI=ON \
-DJPEGXL_FORCE_SYSTEM_GTEST=ON ..
```

Check the output to see if any dependencies were missed and need to be installed. Adding `-G Ninja` may be helpful, but on my computer, Ninja was selected by default. Remember that package changes must be done from the MSYS environment. Then exit all MSYS2 terminals and restart the build environment.

If all went well, you may now run `cmake` to build `libjxl`:

```bash
cmake --build .
```

Do not be alarmed by the compiler warnings. They are a caused by differences between gcc/g++ and clang. The build should complete successfully. Then `cjxl`, `djxl`, `jxlinfo`, and others can be run from within the build environment. Moving them into the native Windows environment requires resolving `dll` issues that are beyond the scope of this document.

## The `clang` Compiler

To use the `clang` compiler, install the packages that correspond with the environment you wish to use. Remember to make package changes from within the MSYS environment.

```
mingw-w64-i686-clang
mingw-w64-i686-clang-tools-extra
mingw-w64-i686-clang-compiler-rt
mingw-w64-x86_64-clang
mingw-w64-x86_64-clang-tools-extra
mingw-w64-x86_64-clang-compiler-rt
mingw-w64-ucrt64-x86_64-clang
mingw-w64-ucrt64-x86_64-clang-tools-extra
mingw-w64-ucrt64-x86_64-clang-compiler-rt
```

After the `clang` compiler is installed, 'libjxl' can be built with the `./ci.sh` script.

```bash
./ci.sh opt -DBUILD_TESTING=OFF -DJPEGXL_ENABLE_BENCHMARK=OFF \
-DJPEGXL_ENABLE_MANPAGES=OFF -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \
-DJPEGXL_FORCE_SYSTEM_GTEST=ON
```

On my computer, `doxygen` packages needed to be installed to proceed with building. Use `pacman -Ss doxygen` to find the packages to install.

## The GIMP Plugin

To build the GIMP plugin, install the relevant `gimp` package. This will also install dependencies. Again, perform package management tasks from only the MSYS environment. Then restart the build environment.

```bash
pacman -S mingw-w64-i686-gimp
pacman -S mingw-w64-x86_64-gimp
pacman -S mingw-w64-ucrt-x86_64-gimp
```

If `clang` is installed, you can use the `./ci.sh` script to build. Otherwise, navigate to the build directory to reconfigure and build with `cmake`.

```bash
cd build
rm -r C*
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DBUILD_TESTING=OFF -DJPEGXL_ENABLE_BENCHMARK=OFF \
-DJPEGXL_ENABLE_PLUGINS=ON -DJPEGXL_ENABLE_MANPAGES=OFF \
-DJPEGXL_FORCE_SYSTEM_BROTLI=ON \
-DJPEGXL_FORCE_SYSTEM_GTEST=ON ..
```

Fortunately, the plugin works without installing `dll` files. To test the plugin:

1. [Download](https://www.gimp.org/downloads/) and install the stable version of GIMP (currently 2.10.24).

2. Create a new folder: `C:\Program Files\GIMP 2\lib\gimp\2.0\plug-ins\file-jxl`

3. Copy `build/plugins/gimp/file-jxl.exe` to the new folder.
Original file line number Diff line number Diff line change
@@ -1,15 +1,19 @@
# Developing on 64-bit Windows
# Developing on Windows with Visual Studio 2019

These instructions assume an up-to-date Windows 10 (e.g. build 19041.928) with
Microsoft Visual Studio 2019 (e.g. Version 16.9.0 Preview 4.0) installed. If
unavailable, please use the [Docker container](developing_in_docker.md).
**Microsoft Visual Studio 2019** (e.g. Version 16.9.0 Preview 4.0) installed. If
unavailable, please use another build environment:

* [Docker container](developing_in_docker.md)
* [MSYS2 on Windows](developing_in_windows_msys.md)
* [Crossroad on Linux](developing_with_crossroad.md) (cross compilation for Windows)

## Minimum build dependencies

Apart from the dependencies in third_party, some of the tools use external
dependencies that need to be installed in your system first.

Please [install vcpkg](https://vcpkg.readthedocs.io/en/latest/examples/installing-and-using-packages/)
Please install [vcpkg](https://vcpkg.readthedocs.io/en/latest/examples/installing-and-using-packages/)
(tested with version 2019.07.18), and use it to install the following libraries:

```
Expand Down Expand Up @@ -83,5 +87,5 @@ vcpkg above.

The project is now ready for use. To build, simply press F7 (or choose
Build All from the Build menu). This writes binaries to
`out/build/x64-Clang-Release/tools`. The main [README.md](README.md) explains
`out/build/x64-Clang-Release/tools`. The main [README.md](../README.md) explains
how to use the encoder/decoder and benchmark binaries.
115 changes: 115 additions & 0 deletions doc/developing_with_crossroad.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
# Cross Compiling for Windows with Crossroad

[Crossroad](https://pypi.org/project/crossroad/) is a tool to set up cross-compilation environments on GNU/Linux distributions. These instructions assume a Debian/Ubuntu system. However, they can likely be adapted to other Linux environments. Since Ubuntu can be run on Windows through WSL, these instruction may be useful for developing directly on Windows.

## Install Crossroad

Crossroad requires tools included with `python3-docutils` and `mingw-w64`. They may be installed using:

```bash
sudo aptitude install python3-docutils mingw-w64
```

The `zstandard` python package is also required, but is not available in the repositories. It may be installed using `pip`.

```bash
pip3 install zstandard
```

After the dependencies are installed, crossroad itself maybe installed with `pip`.

```bash
pip3 install crossroad
```

If there are errors while running crossroad, it may need to be downloaded and installed directly using `setup.py`. Instructions are on the crossroad homepage.

## Update Debian Alternatives

Since `libjxl` uses C++ features that require posix threads, the symlinks used by the Debian alternative system need to be updated:

```bash
sudo update-alternatives --config x86_64-w64-mingw32-g++
```

Select the option that indicates `posix` usage. Repeat for `gcc` and `i686`:

```bash
sudo update-alternatives --config x86_64-w64-mingw32-gcc
sudo update-alternatives --config i686-w64-mingw32-gcc
sudo update-alternatives --config i686-w64-mingw32-g++
```

## Create a New Crossroad Project

Crossroad supports the following platforms:

```
native Native platform (x86_64 GNU/Linux)
android-x86 Generic Android/Bionic on x86
android-mips64 Generic Android/Bionic on MIPS64
android-x86-64 Generic Android/Bionic on x86-64
w64 Windows 64-bit
w32 Windows 32-bit
android-arm64 Generic Android/Bionic on ARM64
android-mips Generic Android/Bionic on MIPS
android-arm Generic Android/Bionic on ARM
```

To begin cross compiling for Windows, a new project needs to be created:

```bash
crossroad w64 [project-name]
```

## Install Dependencies

Since the `gimp` development package is required to build the GIMP plugin and also includes most of the packages required by `libjxl`, install it first.

```bash
crossroad install gimp
```

`gtest` and `brotli` are also required.

```bash
crossroad install gtest brotli
```

If any packages are later found to be missing, you may search for them using:

```bash
crossroad search [...]
```

## Build `libjxl`

Download the source from the libjxl [releases](https://github.com/libjxl/libjxl/releases) page. Alternatively, you may obtain the latest development version with `git`. Run `./deps.sh` to ensure additional third-party dependencies are downloaded. Unfortunately, the script `./ci.sh` does not work with Crossroad, so `cmake` will need to be called directly.

Create a build directory within the source directory. If you haven't already, start your crossroad project and run `cmake`:

```bash
mkdir build
cd build
crossroad w64 libjxl
crossroad cmake -DCMAKE_BUILD_TYPE=Release \
-DBUILD_TESTING=OFF -DJPEGXL_ENABLE_BENCHMARK=OFF \
-DJPEGXL_ENABLE_PLUGINS=ON -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \
-DJPEGXL_FORCE_SYSTEM_GTEST=ON ..
```

Check the output to see if any dependencies were missed and need to be installed. If all went well, you may now run `cmake` to build `libjxl`:

```bash
cmake --build .
```

## Try out the GIMP Plugin

To install and try out out the GIMP plugin:

1. [Download](https://www.gimp.org/downloads/) and install the stable version of GIMP (currently 2.10.24).

2. Create a new folder: `C:\Program Files\GIMP 2\lib\gimp\2.0\plug-ins\file-jxl`

3. Copy `build/plugins/gimp/file-jxl.exe` to the new folder.
184 changes: 184 additions & 0 deletions doc/fuzzing.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
# Fuzzing

Fuzzing is a technique to find potential bugs by providing randomly generated
invalid inputs. To detect potential bugs such as programming errors we use
fuzzing in combination with ASan (Address Sanitizer), MSan (Memory Sanitizer),
UBSan (Undefined Behavior Sanitizer) and asserts in the code. An invalid input
will likely produce a decoding error (some API function returning error), which
is absolutely not a problem, but what it should not do is access memory out of
bounds, use uninitialized memory or hit a false assert condition.

## Automated Fuzzing with oss-fuzz

libjxl fuzzing is integrated into [oss-fuzz](https://github.com/google/oss-fuzz)
as the project `libjxl`. oss-fuzz regularly runs the fuzzers on the `main`
branch and reports bugs into their bug tracker which remains private until the
bugs are fixed in main.

## Fuzzer targets

There are several fuzzer executable targets defined in the `tools/` directory
to fuzz different parts of the code. The main one is `djxl_fuzzer`, which uses
the public C decoder API to attempt to decode an image. The fuzzer input is not
directly the .jxl file, the last few bytes of the fuzzer input are used to
decide *how* will the API be used (if preview is requested, the pixel format
requested, if the .jxl input data is provided altogether, etc) and the rest of
the fuzzer input is provided as the .jxl file to the decoder. Some bugs might
reproduce only if the .jxl input is decoded in certain way.

The remaining fuzzer targets execute a specific portion the codec that might be
easier to fuzz independently from the whole codec.

## Reproducing fuzzer bugs

A fuzzer target, like `djxl_fuzzer` accepts as a parameter one or more files
that will be used as inputs. This runs the fuzzer program in test-only mode
where no new inputs are generated and only the provided files are tested. This
is the easiest way to reproduce a bug found by the fuzzer using the generated
test case from the bug report.

oss-fuzz uses a specific compiler version and flags, and it is built using
Docker. Different compiler versions will have different support for detecting
certain actions as errors, so we want to reproduce the build from oss-fuzz as
close as possible. To reproduce the build as generated by oss-fuzz there are a
few helper commands in `ci.sh` as explained below.

### Generate the gcr.io/oss-fuzz/libjxl image

First you need the ossfuzz libjxl builder image. This is the base oss-fuzz
builder image with a few dependencies installed. To generate it you need to
check out the oss-fuzz project and build it:

```bash
git clone https://github.com/google/oss-fuzz.git ~/oss-fuzz
cd ~/oss-fuzz
sudo infra/helper.py build_image libjxl
```

This will create the `gcr.io/oss-fuzz/libjxl` docker image. You can check if it
was created verifying that it is listed in the output of the `sudo docker image
ls` command.

### Build the fuzzer targets with oss-fuzz

To build the fuzzer targets from the current libjxl source checkout, use the
`./ci.sh ossfuzz_msan` command for MSan, `./ci.sh ossfuzz_asan` command for ASan
or `./ci.sh ossfuzz_ubsan` command for UBSan. All the `JXL_ASSERT` and
`JXL_DASSERT` calls are enabled in all the three modes. These ci.sh helpers will
reproduce the oss-fuzz docker call to build libjxl mounting the current source
directory into the Docker container. Ideally you will run this command in a
different build directory separated from your regular builds.

For example, for MSan builds run:

```bash
BUILD_DIR=build-fuzzmsan ./ci.sh ossfuzz_msan
```

After this, the fuzzer program will be generated in the build directory like
for other build modes: `build-fuzzmsan/tools/djxl_fuzzer`.

### Iterating changes with oss-fuzz builds

After modifying the source code to fix the fuzzer-found bug, or to include more
debug information, you can rebuild only a specific fuzzer target to save on
rebuilding time and immediately run the test case again. For example, for
rebuilding and testing only `djxl_fuzzer` in MSan mode we can run:

```bash
BUILD_DIR=build-fuzzmsan ./ci.sh ossfuzz_msan djxl_fuzzer && build-fuzzmsan/tools/djxl_fuzzer path/to/testcase.bin
```

When MSan and ASan fuzzers fail they will print a stack trace at the point where
the error occurred, and some related information. To make these these stack
traces useful we need to convert the addresses to function names and source file
names and lines, which is done with the "symbolizer". For UBSan to print a stack
trace we need to set the `UBSAN_OPTIONS` environment variables when running the
fuzzer.

Set the following environment variables when testing the fuzzer binaries. Here
`clang` should match the compiler version used by the container, you can pass a
different compiler version in the following example by first installing the
clang package for that version outside the container and using `clang-NN`
(for example `clang-11`) instead of `clang` in the following commands:

```bash
symbolizer=$($(realpath $(which clang)) -print-prog-name=llvm-symbolizer)
export MSAN_SYMBOLIZER_PATH="${symbolizer}"
export UBSAN_SYMBOLIZER_PATH="${symbolizer}"
export ASAN_SYMBOLIZER_PATH="${symbolizer}"
export ASAN_OPTIONS=detect_leaks=1
export UBSAN_OPTIONS=print_stacktrace=1
```

Note: The symbolizer binary must be a program called `llvm-symbolizer`, any
other file name will fail. There are normally symlinks already installed with
the right name which the `-print-prog-name` would print.

## Running the fuzzers locally

Running the fuzzer targets in fuzzing mode can be achieved by running them with
no parameters, or better with a parameter with the path to a *directory*
containing a seed of files to use as a starting point. Note that passing a
directory is considered a corpus to use for fuzzing while passing a file is
considered an input to evaluate. Multi-process fuzzing is also supported. For
details about all the fuzzing options run:

```bash
build-fuzzmsan/tools/djxl_fuzzer -help=1
```

## Writing fuzzer-friendly code

Fuzzing on itself can't find programming bugs unless an input makes the program
perform an invalid operation (read/write out of bounds, perform an undefined
behavior operation, etc). You can help the fuzzer find invalid situations by
adding asserts:

* `JXL_ASSERT()` is enabled in Release mode by default. It can be disabled
with `-DJXL_ENABLE_ASSERT=0` but the intention is that it will run for all
the users in released code. If performance of the check is not an issue (like
checks done once per image, once per channel, once per group, etc) a
JXL_ASSERT is appropriate. A failed assert is preferable to an out of bounds
write.

* `JXL_DASSERT()` is only enabled in Debug builds, which includes all the ASan,
MSan and UBSan builds. Performance of these checks is not an issue if kept
within reasonable limits (automated msan/asan test should finish withing 1
hour for example). Fuzzing is more effective when the given input runs
faster, so keep that in mind when adding a complex DASSERT that runs multiple
times per output pixel.

* For MSan builds it is also possible to specify that certain values must be
initialized. This is automatic for values that are used to make decisions
(like when used in an `if` statement or in the ternary operator condition)
but those checks can be made explicit for image data using the
`JXL_CHECK_IMAGE_INITIALIZED(image, rect)` macro. This helps document and
check (only in MSan builds) that a given portion of the image is expected to
be initialized, allowing to catch errors earlier in the process.

## Dealing with use-of-uninitialized memory

In MSan builds it is considered an error to *use* uninitialized memory. Using
the memory normally requires something like a decision / branch based on the
uninitialized value, just running `memcpy()` or simple arithmetic over
uninitialized memory is not a problem. Notably, computing `DemoteTo()`,
`NearestInt()` or similar expressions that create a branch based on the value of
the uninitialized memory will trigger an MSan error.

In libjxl we often run vectorized operations over a series of values, rounding
up to the next multiple of a vector size, thus operating over uninitialized
values past the end of the requested region. These values are part of the image
padding but are not initialized. This behavior would not create an MSan error
unless the processing includes operations like `NearestInt()`. For such cases
the preferred solution is to use `msan::UnpoisonMemory` over the portion of
memory of the last SIMD vector before processing, and then running
`msan::PoisonMemory` over the corresponding value in the output side. A note
including why this is safe to do must be added, for example if the processing
doesn't involve any cross-lane computation.

Initializing padding memory in MSan builds is discouraged because it may hide
bugs in functions that weren't supposed to read from the padding. Initializing
padding memory in all builds, including Release builds, would mitigate the
MSan potential security issue but it would hide the logic bug for a longer time
and potentially incur in a performance hit.
262 changes: 262 additions & 0 deletions doc/release.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,262 @@
# libjxl release process

This guide documents the release process for the libjxl project.

libjxl follows the [semantic versioning](https://semver.org/spec/v2.0.0.html)
specification for released versions. Releases are distributed as tags in the git
repository with the semantic version prefixed by the letter "v". For example,
release version "0.3.7" will have a git tag "v0.3.7".

The public API is explicitly defined as C headers in the `lib/include`
directory, normally installed in your include path. All other headers are
internal API and are not covered by the versioning rules.

## Development and release workflow

New code development is performed on the `main` branch of the git repository.
Pre-submit checks enforce minimum build and test requirements for new patches
that balance impact and test latency, but not all checks are performed before
pull requests are merged. Several slower checks only run *after* the code has
been merged to `main`, resulting in some errors being detected hours after the
code is merged or even days after in the case of fuzzer-detected bugs.

Release tags are cut from *release branches*. Each MAJOR.MINOR version has its
own release branch, for example releases `0.5`, `0.5.1`, `0.5.2`, ... would have
tags `v0.5`, `v0.5.1`, `v0.5.2`, ... on commits from the `v0.5.x` branch.
`v0.5.x` is a branch name, not a tag name, and doesn't represent a released
version since semantic versioning requires that the PATCH is a non-negative
number. Released tags don't each one have their own release branch, all releases
from the same MAJOR.MINOR version will share the same branch.

The main purpose of the release branch is to stabilize the code before a
release. This involves including fixes to existing bugs but **not** including
new features. New features often come with new bugs which take time to fix, so
having a release branch allows us to cherry-pick *bug fixes* from the `main`
branch into the release branch without including the new *features* from `main`.
For this reason it is important to make small commits in `main` and separate bug
fixes from new features.

After the initial minor release (`M.N`, for example `0.5.0` or just `0.5`) the
release branch is used to continue to cherry-pick fixes to be included in a
patch release, for example a version `0.5.1` release. Patch fixes are only meant
to fix security bugs or other critical bugs that can't wait until the next major
or minor release.

Release branches *may* continue to be maintained even after the next minor or
major version has been released to support users that can't update to a newer
minor release. In that case, the same process applies to all the maintained
release branches.

A release branch with specific cherry-picks from `main` means that the release
code is actually a version of the code that never existed in the `main` branch,
so it needs to be tested independently. Pre-submit and post-submit tests run on
release branches (branches matching `v*.*.x`) but extra manual checks should be
performed before a release, specially if multiple bug fixes interact with each
other. Take this into account when selecting which commits to include in a
release. The objective is to have a stable version that can be used without
problems for months. Having the latest improvements at the time the release tag
is created is a non-goal.

## Creating a release branch

A new release branch is needed before creating a new major or minor release,
that is, a new release where the MAJOR or MINOR numbers are increased. Patch
releases, where only the PATCH number is increased, reuse the branch from the
previous release of the same MAJOR and MINOR numbers.

The following instructions assume that you followed the recommended [libjxl git
setup](developing_in_github.md) where `origin` points to the upstream
libjxl/libjxl project, otherwise use the name of your upstream remote repository
instead of `origin`.

The release branch is normally created from the latest work in `main` at the
time the branch is created, but it is possible to create the branch from an
older commit if the current `main` is particularly unstable or includes commits
that were not intended to be included in the release. The following example
creates the branch `v0.5.x` from the latest commit in main (`origin/main`), if a
different commit is to be used then replace `origin/main` with the SHA of that
commit. Change the `v0.5.x` branch name to the one you are creating.

```bash
git fetch origin main
git push git@github.com:libjxl/libjxl.git origin/main:refs/heads/v0.5.x
```

Here we use the SSH URL explicitly since you are pushing to the `libjxl/libjxl`
project directly to a branch there. If you followed the guide `origin` will have
the HTTPS URL which wouldn't normally let you push since you wouldn't be
authenticated. The `v*.*.x` branches are [GitHub protected
branches](https://docs.github.com/en/github/administering-a-repository/defining-the-mergeability-of-pull-requests/about-protected-branches)
in our repository, however you can push to a protected branch when *creating* it
but you can't directly push to it after it is created. To include more changes
in the release branch see the "Cherry-picking fixes to a release" section below.

## Creating a merge label

We use GitHub labels in Pull Requests to keep track of the changes that should
be merged into a given release branch. For this purpose create a new label for
each new MAJOR.MINOR release branch called `merge-MAJOR.MINOR`, for example,
`merge-0.5`.

In the [edit labels](https://github.com/libjxl/libjxl/issues/labels) page, click
on "New label" and create the label. Pick your favorite color.

Labels are a GitHub-only concept and are not represented in git. You can add the
label to a Pull Request even after it was merged, whenever it is decided that
the Pull Request should be included in the given release branch. Adding the
label doesn't automatically merge it to the release branch.

## Update the versioning number

The version number (as returned by `JxlDecoderVersion`) in the source code in
`main` must match the semantic versioning of a release. After the release
branch is created the code in `main` will only be included in the next major
or minor release. Right after a release branch update the version targeting the
next release. Artifacts from `main` should include the new (unreleased) version,
so it is important to update it. For example, after the `v0.5.x` branch is
created from main, you should update the version on `main` to `0.6`.

To help update it, run this helper command (in a Debian-based system):

```bash
./ci.sh bump_version 0.6
```

This will update the version in the following files:

* `lib/CMakeLists.txt`
* `lib/lib.gni`, automatically updated with `tools/build_cleaner.py --update`.
* `debian/changelog` to create the Debian package release with the new version.
Debian changelog shouldn't repeat the library changelog, instead it should
include changes to the packaging scripts.

## Cherry-pick fixes to a release

After a Pull Request that should be included in a release branch has been merged
to `main` it can be cherry-picked to the release branch. Before cherry-picking a
change to a release branch it is important to check that it doesn't introduce
more problems, in particular it should run for some time in `main` to make sure
post-submit tests and the fuzzers run on it. Waiting for a day is a good idea.

Most of the testing is done on the `main` branch, so be careful with what
commits are cherry-picked to a branch. Refactoring code is often not a good
candidate to cherry-pick.

To cherry-pick a single commit to a release branch (in this example to `v0.5.x`)
you can run:

```bash
git fetch origin
git checkout origin/v0.5.x -b merge_to_release
git cherry-pick -x SHA_OF_MAIN_COMMIT
# -x will annotate the cherry-pick with the original SHA_OF_MAIN_COMMIT value.
# If not already mentioned in the original commit, add the original PR number to
# the commit, for example add "(cherry picked from PR #NNNN)".
git commit --amend
```

The `SHA_OF_MAIN_COMMIT` is the hash of the commit as it landed in main. Use
`git log origin/main` to list the recent main commits and their hashes.

Making sure that the commit message on the cherry-picked commit contains a
reference to the original pull request (like `#NNNN`) is important. It creates
an automatic comment in the original pull request notifying that it was
mentioned in another commit, helping keep track of the merged pull requests. If
the original commit was merged with the "Squash and merge" policy it will
automatically contain the pull request number on the first line, if this is not
the case you can amend the commit message of the cherry-pick to include a
reference.

Multiple commits can be cherry-picked and tested at once to save time. Continue
running `git cherry-pick` and `git commit --amend` multiple times for all the
commits you need to cherry-pick, ideally in the same order they were merged on
the `main` branch. At the end you will have a local branch with multiple commits
on top of the release branch.

Finally, upload your changes to *your fork* like normal, except that when
creating a pull request select the desired release branch as a target:

```bash
git push myfork merge_to_release
```

If you used the [guide](developing_in_github.md) `myfork` would be `origin` in
that example. Click on the URL displayed, which will be something like

`https://github.com/mygithubusername/libjxl/pull/new/merge_to_release`

In the "Open a pull request" page, change the drop-down base branch from
"base: main" (the default) to the release branch you are targeting.

The pull request approval and pre-submit rules apply as with normal pull
requests to the `main` branch.

**Important:** When merging multiple cherry-picks use "Rebase and merge" policy,
not the squash one since otherwise you would discard the individual commit
message references from the git history in the release branch.

## Publishing a release

Once a release tag is created it must not be modified, so you need to prepare
the changes before creating the release. Make sure you checked the following:

* The semantic version number in the release branch (see `lib/CMakeLists.txt`)
matches the number you intend to release, all three MAJOR, MINOR and PATCH
should match. Otherwise send a pull request to the release branch to
update them.

* The GitHub Actions checks pass on the release branch. Look for the green
tick next to the last commit on the release branch. This should be visible
on the branch page, for example: https://github.com/libjxl/libjxl/tree/v0.5.x

* There no open fuzzer-found bugs for the release branch. The most effective
way is to [run the fuzzer](fuzzing.md) on the release branch for a while. You
can seed the fuzzer with corpus generated by oss-fuzz by [downloading
it](https://google.github.io/oss-fuzz/advanced-topics/corpora/#downloading-the-corpus),
for example `djxl_fuzzer` with libFuzzer will use:
gs://libjxl-corpus.clusterfuzz-external.appspot.com/libFuzzer/libjxl_djxl_fuzzer

* Manually check that images encode/decode ok.

* Manually check that downstream projects compile with our code. Sometimes
bugs on build scripts are only detected when other projects try to use our
library. For example, test compiling
[imagemagick](https://github.com/ImageMagick/ImageMagick) and Chrome.

A [GitHub
"release"](https://docs.github.com/en/github/administering-a-repository/releasing-projects-on-github/about-releases)
consists of two different concepts:

* a git "tag": this is a name (`v` plus the semantic version number) with a
commit hash associated, defined in the git repository. Most external projects
will use git tags or HTTP URLs to these tags to fetch the code.

* a GitHub "release": this is a GitHub-only concept and is not represented in
git other than by having a git tag associated with the release. A GitHub
release has a given source code commit SHA associated (through the tag) but
it *also* contains release notes and optional binary files attached to the
release.

Releases from the older GitLab repository only have a git tag in GitHub, while
newer releases have both a git tag and a release entry in GitHub.

To publish a release open the [New Release
page](https://github.com/libjxl/libjxl/releases/new) and follow these
instructions:

* Set the "Tag version" as "v" plus the semantic version number. Omit the ".0"
when the PATCH version is 0, for example use "v0.5" or "v0.5.1" but not
"v0.5.0".

* Select the "Target" as your release branch. For a "v0.5" release tag you
would use the "v0.5.x" branch.

* Use the version number as the release title.

* Copy-paste the relevant section of the [CHANGELOG.md](../CHANGELOG.md) to the
release notes into the release notes. Add any other information pertaining
the release itself that are not included in the CHANGELOG.md, although prefer
to include those in the CHANGELOG.md file. You can switch to the Preview tab
to see the results.

* Finally click "Publish release" and go celebrate with the team. 🎉
55 changes: 55 additions & 0 deletions doc/software_support.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# JPEG XL software support

This document attempts to keep track of software that is using libjxl to support JPEG XL.
This list serves several purposes:

- thank/acknowledge other projects for integrating jxl support
- point end-users to software that can read/write jxl
- keep track of the adoption status of jxl
- in case of a (security) bug in libjxl, it's easier to see who might be affected and check if they are updated (in case they use static linking)

Please add missing software to this list.

## Browsers

- Chromium: behind a flag since version 91, [tracking bug](https://bugs.chromium.org/p/chromium/issues/detail?id=1178058)
- Firefox: behind a flag since version 90, [tracking bug](https://bugzilla.mozilla.org/show_bug.cgi?id=1539075)
- Safari: not supported, [tracking bug](https://bugs.webkit.org/show_bug.cgi?id=208235)
- Edge: behind a flag since version 91, start with `.\msedge.exe --enable-features=JXL`
- Opera: behind a flag since version 77.

## Image libraries

- [ImageMagick](https://imagemagick.org/): supported since 7.0.10-54
- [libvips](https://libvips.github.io/libvips/): supported since 8.11
- [Imlib2](https://github.com/alistair7/imlib2-jxl)

## OS-level support / UI frameworks / file browser plugins

- Qt / KDE: [plugin available](https://github.com/novomesk/qt-jpegxl-image-plugin)
- GDK-pixbuf: plugin available in libjxl repo
- [gThumb](https://ubuntuhandbook.org/index.php/2021/04/gthumb-3-11-3-adds-jpeg-xl-support/)
- [MacOS viewer/QuickLook plugin](https://github.com/yllan/JXLook)
- [Windows Imaging Component](https://github.com/mirillis/jpegxl-wic)
- [Windows thumbnail handler](https://github.com/saschanaz/jxl-winthumb)
- [OpenMandriva Lx (since 4.3 RC)](https://www.openmandriva.org/en/news/article/openmandriva-lx-4-3-rc-available-for-testing)
- [KaOS (since 2021.06)](https://news.itsfoss.com/kaos-2021-06-release/)

## Image editors

- GIMP: plugin available in libjxl repo, no official support, [tracking bug](https://gitlab.gnome.org/GNOME/gimp/-/issues/4681)
- Photoshop: no plugin available yet, no official support yet

## Image viewers

- [XnView](https://www.xnview.com/en/)
- [ImageGlass](https://imageglass.org/)
- Any viewer based on Qt, KDE, GDK-pixbuf, ImageMagick, libvips or imlib2 (see above)
- Qt viewers: gwenview, digiKam, KolourPaint, KPhotoAlbum, LXImage-Qt, qimgv, qView, nomacs, VookiImageViewer, PhotoQt
- GTK viewers: Eye of Gnome (eog), gThumb, Geeqie

## Online tools

- [Squoosh](https://squoosh.app/)
- [Cloudinary](https://cloudinary.com/blog/cloudinary_supports_jpeg_xl)
- [MConverter](https://mconverter.eu/)
57 changes: 47 additions & 10 deletions examples/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,54 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# Example project using libjxl.

cmake_minimum_required(VERSION 3.10)

project(SAMPLE_LIBJXL LANGUAGES C CXX)

# Use pkg-config to find libjxl.
find_package(PkgConfig)
pkg_check_modules(Jxl REQUIRED IMPORTED_TARGET libjxl)
pkg_check_modules(JxlThreads REQUIRED IMPORTED_TARGET libjxl_threads)

# Build the example encoder/decoder binaries using the default shared libraries
# installed.
add_executable(decode_oneshot decode_oneshot.cc)
target_link_libraries(decode_oneshot jxl_dec jxl_threads)
target_link_libraries(decode_oneshot PkgConfig::Jxl PkgConfig::JxlThreads)

add_executable(encode_oneshot encode_oneshot.cc)
target_link_libraries(encode_oneshot jxl jxl_threads)
target_link_libraries(encode_oneshot PkgConfig::Jxl PkgConfig::JxlThreads)

add_executable(jxlinfo jxlinfo.c)
target_link_libraries(jxlinfo jxl)

if(NOT ${SANITIZER} STREQUAL "none")
# Linking a C test binary with the C++ JPEG XL implementation when using
# address sanitizer is not well supported by clang 9, so force using clang++
# for linking this test if a sanitizer is used.
set_target_properties(jxlinfo PROPERTIES LINKER_LANGUAGE CXX)
endif() # SANITIZER != "none"
target_link_libraries(jxlinfo PkgConfig::Jxl)


# Building a static binary with the static libjxl dependencies. How to load
# static library configs from pkg-config and how to build static binaries
# depends on the platform, and building static binaries in general has problems.
# If you don't need static binaries you can remove this section.
add_library(StaticJxl INTERFACE IMPORTED GLOBAL)
set_target_properties(StaticJxl PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${Jxl_STATIC_INCLUDE_DIR}"
INTERFACE_COMPILE_OPTIONS "${Jxl_STATIC_CFLAGS_OTHER}"
INTERFACE_LINK_LIBRARIES "${Jxl_STATIC_LDFLAGS}"
)
add_library(StaticJxlThreads INTERFACE IMPORTED GLOBAL)
set_target_properties(StaticJxlThreads PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${JxlThreads_STATIC_INCLUDE_DIR}"
INTERFACE_COMPILE_OPTIONS "${JxlThreads_STATIC_CFLAGS_OTHER}"
# libgcc uses weak symbols for pthread which means that -lpthread is not
# linked when compiling a static binary. This is a platform-specific fix for
# that.
INTERFACE_LINK_LIBRARIES
"${JxlThreads_STATIC_LDFLAGS} -Wl,--whole-archive -lpthread -Wl,--no-whole-archive"
)

add_executable(decode_oneshot_static decode_oneshot.cc)
target_link_libraries(decode_oneshot_static
-static StaticJxl StaticJxlThreads)

add_executable(encode_oneshot_static encode_oneshot.cc)
target_link_libraries(encode_oneshot_static
-static StaticJxl StaticJxlThreads)
5 changes: 2 additions & 3 deletions examples/encode_oneshot.cc
Original file line number Diff line number Diff line change
Expand Up @@ -162,13 +162,12 @@ bool EncodeJxlOneshot(const std::vector<float>& pixels, const uint32_t xsize,

JxlPixelFormat pixel_format = {3, JXL_TYPE_FLOAT, JXL_NATIVE_ENDIAN, 0};

JxlBasicInfo basic_info = {};
JxlBasicInfo basic_info;
JxlEncoderInitBasicInfo(&basic_info);
basic_info.xsize = xsize;
basic_info.ysize = ysize;
basic_info.bits_per_sample = 32;
basic_info.exponent_bits_per_sample = 8;
basic_info.alpha_exponent_bits = 0;
basic_info.alpha_bits = 0;
basic_info.uses_original_profile = JXL_FALSE;
if (JXL_ENC_SUCCESS != JxlEncoderSetBasicInfo(enc.get(), &basic_info)) {
fprintf(stderr, "JxlEncoderSetBasicInfo failed\n");
Expand Down
19 changes: 19 additions & 0 deletions examples/examples.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Copyright (c) the JPEG XL Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

add_executable(decode_oneshot ${CMAKE_CURRENT_LIST_DIR}/decode_oneshot.cc)
target_link_libraries(decode_oneshot jxl_dec jxl_threads)
add_executable(encode_oneshot ${CMAKE_CURRENT_LIST_DIR}/encode_oneshot.cc)
target_link_libraries(encode_oneshot jxl jxl_threads)

add_executable(jxlinfo ${CMAKE_CURRENT_LIST_DIR}/jxlinfo.c)
target_link_libraries(jxlinfo jxl)

if(NOT ${SANITIZER} STREQUAL "none")
# Linking a C test binary with the C++ JPEG XL implementation when using
# address sanitizer is not well supported by clang 9, so force using clang++
# for linking this test if a sanitizer is used.
set_target_properties(jxlinfo PROPERTIES LINKER_LANGUAGE CXX)
endif() # SANITIZER != "none"
Loading