-
Notifications
You must be signed in to change notification settings - Fork 1
462 lines (460 loc) · 27.2 KB
/
asset_zcodecs.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
name: asset_zcodecs
on:
workflow_dispatch:
permissions:
contents: write
jobs:
prepare:
runs-on: ubuntu-24.04
steps:
- uses: actions/cache@v4 # the post step for this will upload tar if not hit
id: cache
with:
enableCrossOsArchive: true
path: zcodecs.tar
key: "asset_zcodecs_12.0.0"
- name: Build
if: steps.cache.outputs.cache-hit != 'true'
run: |
mkdir zcodecs
cd zcodecs
# ========== ect
mkdir ect
cd ect
curl -L https://github.com/fhanau/Efficient-Compression-Tool/archive/9aabc23d73899ae55c1de292592fed6eb6217f66.tar.gz | tar -zx --strip-components 1 # 0.9.5
curl -L https://github.com/pnggroup/libpng/archive/f135775ad4e5d4408d2e12ffcc71bb36e6b48551.tar.gz | tar -zx --strip-components 1 -C src/libpng
curl -L https://github.com/mozilla/mozjpeg/archive/6c9f0897afa1c2738d7222a0a9ab49e8b536a267.tar.gz | tar -zx --strip-components 1 -C src/mozjpeg # 4.1.5
mv src/* ./
sed -i.bak -e 's|set(PNG_BUILD_ZLIB |#|' optipng/CMakeLists.txt
sed -i.bak -e 's|message(FATAL_ERROR |#|' -e 's|option(PNG_SUPPORTED |option(PNG_SUPPORTED "" FALSE) #|' mozjpeg/CMakeLists.txt
sed -i.bak -e 's|find_package(ZLIB REQUIRED)|#|' libpng/CMakeLists.txt
sed -i.bak -e 's|<float.h>|<float.h>\n#include <math.h>\n|' libpng/pngpriv.h # https://github.com/pnggroup/libpng/commit/893b8113f04d408cc6177c6de19c9889a48faa24
sed -i.bak -e 's|-Ofast|-O3 -ffast-math|' CMakeLists.txt
echo "" > pngusr.h # the ect disabled some libpng features to reduce size, but other programs require full-featured libpng
sed -i.bak -e 's|option(ENABLE_SHARED |option(ENABLE_SHARED "" FALSE) #|' mozjpeg/CMakeLists.txt
sed -i.bak -e 's|configure_file(win/${INST_ID}/projectTargets-release.cmake.in|#|' -e 's|win/${CMAKE_PROJECT_NAME}Targets-release.cmake @ONLY)|#|' mozjpeg/cmakescripts/BuildPackages.cmake
sed -i.bak -e 's|int main(|extern "C" int cmd_ect_main(|' main.cpp
cd ..
# ========== webp
mkdir webp
cd webp
curl -L https://github.com/webmproject/libwebp/archive/2e81017c7a345f687223086cbc177a8459a18b52.tar.gz | tar -zx --strip-components 1 # 20240901 > 1.4.0
sed -i.bak -e 's|int main(|int cmd_webpinfo_main(|' examples/webpinfo.c
sed -i.bak -e 's|int main(|int cmd_cwebp_main(|' examples/cwebp.c
sed -i.bak -e 's|int main(|int cmd_dwebp_main(|' examples/dwebp.c
sed -i.bak -e 's|int main(|int cmd_gif2webp_main(|' examples/gif2webp.c
sed -i.bak -e 's|int main(|int cmd_img2webp_main(|' examples/img2webp.c
sed -i.bak -e 's|int main(|int cmd_webpmux_main(|' examples/webpmux.c
cd ..
# ========== jpegxl
mkdir jpegxl
cd jpegxl
curl -L https://github.com/libjxl/libjxl/archive/f28befab0b0ef113ffdca3786757737bce01d4c7.tar.gz | tar -zx --strip-components 1 # 20240901 > 0.10.3
sed -i.bak -e 's|download_github testdata|#|' deps.sh
./deps.sh
rm -rf downloads third_party/skcms/profiles/*
sed -i.bak -e 's|string(APPEND CMAKE_EXE_LINKER_FLAGS " -static")||' -e 's|string(APPEND CMAKE_EXE_LINKER_FLAGS " -static-libgcc -static-libstdc++")||' -e 's|CMAKE_USE_PTHREADS_INIT|0|' CMakeLists.txt
sed -i.bak -e 's|int main(|int cmd_jxlinfo_main(|' tools/jxlinfo.c
sed -i.bak -e 's|int main(|extern "C" int cmd_cjxl_main(|' tools/cjxl_main.cc
sed -i.bak -e 's|int main(|extern "C" int cmd_djxl_main(|' tools/djxl_main.cc
sed -i.bak -e 's|int main(|extern "C" int cmd_cjpegli_main(|' tools/cjpegli.cc
sed -i.bak -e 's|int main(|extern "C" int cmd_djpegli_main(|' tools/djpegli.cc
cd ..
# ========== multi call
cat <<EOF > multicall.cc
#include <stddef.h>
#include <string.h>
#include <stdio.h>
#if defined(WIN32) || defined(_WIN32)
#define PATH_SEPARATOR '\\\\'
#else
#define PATH_SEPARATOR '/'
#endif
extern "C" {
int cmd_ect_main(int argc, const char* argv[]);
int cmd_webpinfo_main(int argc, const char* argv[]);
int cmd_cwebp_main(int argc, const char* argv[]);
int cmd_dwebp_main(int argc, const char* argv[]);
int cmd_gif2webp_main(int argc, const char* argv[]);
int cmd_img2webp_main(int argc, const char* argv[]);
int cmd_webpmux_main(int argc, const char* argv[]);
int cmd_jxlinfo_main(int argc, const char* argv[]);
int cmd_cjxl_main(int argc, const char* argv[]);
int cmd_djxl_main(int argc, const char* argv[]);
int cmd_cjpegli_main(int argc, const char* argv[]);
int cmd_djpegli_main(int argc, const char* argv[]);
}
int main(int argc, const char *argv[]) {
for (int i = 0; argc != 0 && i != 2; i++) {
const char *argv0 = strrchr(argv[0], PATH_SEPARATOR);
if (argv0 == NULL) argv0 = argv[0]; else argv0++;
if (strcmp(argv0, "ect") == 0) return cmd_ect_main(argc, argv);
if (strcmp(argv0, "webpinfo") == 0) return cmd_webpinfo_main(argc, argv);
if (strcmp(argv0, "cwebp") == 0) return cmd_cwebp_main(argc, argv);
if (strcmp(argv0, "dwebp") == 0) return cmd_dwebp_main(argc, argv);
if (strcmp(argv0, "gif2webp") == 0) return cmd_gif2webp_main(argc, argv);
if (strcmp(argv0, "img2webp") == 0) return cmd_img2webp_main(argc, argv);
if (strcmp(argv0, "webpmux") == 0) return cmd_webpmux_main(argc, argv);
if (strcmp(argv0, "jxlinfo") == 0) return cmd_jxlinfo_main(argc, argv);
if (strcmp(argv0, "cjxl") == 0) return cmd_cjxl_main(argc, argv);
if (strcmp(argv0, "djxl") == 0) return cmd_djxl_main(argc, argv);
if (strcmp(argv0, "cjpegli") == 0) return cmd_cjpegli_main(argc, argv);
if (strcmp(argv0, "djpegli") == 0) return cmd_djpegli_main(argc, argv);
argv++;
argc--;
}
puts("applets: ect webpinfo cwebp dwebp gif2webp img2webp webpmux jxlinfo cjxl djxl cjpegli djpegli");
return 0;
}
EOF
mkdir pgo_res_bak
curl \
-o pgo_res_bak/4.tar.gz -L https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz \
-o pgo_res_bak/5.tar.gz -L https://github.com/NixOS/patchelf/archive/refs/tags/0.18.0.tar.gz \
-o pgo_res_bak/2.png -L https://user-images.githubusercontent.com/634063/202742985-bb3b3b94-8aca-404a-8d8a-fd6a6f030672.png \
-o pgo_res_bak/3.jpg -L https://raw.githubusercontent.com/WongKinYiu/yolov7/a207844b1ce82d204ab36d87d496728d3d2348e7/figure/yolov7_city.jpg \
--parallel
gzip -d pgo_res_bak/*.tar.gz
truncate --size=1234KiB 5.tar
cat pgo_res_bak/*.tar pgo_res_bak/2.png > pgo_res_bak/1.bin
rm -rf pgo_res_bak/*.tar pgo_res_bak/*.gz
gzip -k pgo_res_bak/1.bin
cat <<'EOF' > utils.sh
ninja_targets(){
cat build/build.ninja | grep $1 | sed -e 's|\$||g' -e 's/|/ /g' | cut -d " " -f 4- | tr " " "\n" | grep -E "\.[^\\/]+$" # get targets built by $1, fix msys2 paths like "D$:/a.o", remove target head and '|' char, exclude targets without extension name
}
run_profiling(){
pgo_i=1
rm -rf *.profraw *.profdata pgo
profile(){ pgo_i=$(awk "BEGIN {print $pgo_i + 1}") ; ./zcodecs $* ;} # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47618#c5
if [ $1 = llvm ]; then
profile(){ pgo_i=$(awk "BEGIN {print $pgo_i + 1}") ; export LLVM_PROFILE_FILE="pgo-$(printf %03d $pgo_i).profraw"; ./zcodecs $* ;}
fi
for i in 1 2; do # small files twice is enough
rm -rf pgo_res
cp -r pgo_res_bak pgo_res
profile ect -2 -zip pgo_res/1.bin # zip compress an easy file + a hard file
profile ect -4 -gzip pgo_res/1.bin.gz # gzip re-compress with higher level, include both compress and decompress
profile cwebp -lossless pgo_res/3.png -o pgo_res/3.webp # to webp lossless
profile cwebp -lossless -sharp_yuv -m 6 pgo_res/2.png -o pgo_res/2.webp
profile cwebp -crop 100 100 2144 1544 pgo_res/3.png -o pgo_res/3.1.webp # to webp lossy
profile dwebp -crop 300 300 2144 1544 pgo_res/3.webp -o pgo_res/3.2.png # webp decode and crop 2
profile img2webp -lossy -sharp_yuv -m 6 pgo_res/2.png pgo_res/3.1.webp pgo_res/3.2.png -o pgo_res/3.0.webp # to webp animated
profile webpinfo pgo_res/3.0.webp
profile cjxl -q 68 -e 10 pgo_res/2.png pgo_res/2.jxl # to jpegxl
profile cjxl -q 85 pgo_res/3.png pgo_res/3.jxl
profile cjxl -q 100 -e 8 pgo_res/3.2.png pgo_res/3.2.jxl # mathematically lossless
profile djxl pgo_res/2.jxl pgo_res/2.jpeg # to normal jpeg
profile jxlinfo pgo_res/2.jxl
profile ect -3 pgo_res/2.jpeg pgo_res/2.png # similar to jpegtran and zopflipng
profile ect -9 pgo_res/2.png pgo_res/3.2.png # high level recompress
profile cjpegli -q 85 pgo_res/3.png pgo_res/3.jpegli.jpeg
profile cjpegli -q 60 pgo_res/2.png pgo_res/2.jpegli.jpeg
profile cjpegli -q 75 pgo_res/3.2.png pgo_res/3.2.jpegli.jpeg
profile djpegli pgo_res/3.jpegli.jpeg pgo_res/3.jpegli.png
done
if [ $1 = llvm ]; then
llvm-profdata merge -output=pgo-0.profdata *.profraw
fi
rm -rf *.profraw
}
EOF
cd ..
tar -cf zcodecs.tar --dereference zcodecs
linux-x64:
needs: prepare
runs-on: ubuntu-24.04 # don't worry, we use debian 12 (docker gcc:14-bookworm) below
steps:
- uses: actions/cache/restore@v4
with:
enableCrossOsArchive: true
path: zcodecs.tar
key: "asset_zcodecs_12.0.0"
- name: Build
shell: docker run -i --network host --privileged --mount type=bind,source=/,destination=/bind gcc:14-bookworm bash -e /bind{0} # github actions "running-jobs-in-a-container" feature is sucks, cause "uses" failed
run: |
cd /bind${{ github.workspace }}
tar -xf zcodecs.tar --strip-components 1
source utils.sh
export LANG=C DEBIAN_FRONTEND=noninteractive # https://unix.stackexchange.com/a/269164 # https://github.com/moby/moby/issues/27988
apt update
apt install -y zip zstd busybox curl cmake ninja-build nasm libgif-dev
for step in profile final; do
export CC="gcc" CXX="g++" CFLAGS="-O3 -fomit-frame-pointer -march=x86-64-v3 -flto -fprofile-update=atomic -fprofile-dir=$(pwd)/pgo"
if [ $step = profile ]; then
export CFLAGS="$CFLAGS -fprofile-generate"
elif [ $step = final ]; then
export CFLAGS="$CFLAGS -fprofile-use"
fi
export CXXFLAGS="$CFLAGS"
# ========== ect
cd ect
rm -rf build
cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \
-DECT_MULTITHREADING=OFF # ect use it's custom zlib, so link to system zlib is impossible
ect_targets=""
ect_targets="$ect_targets $(ninja_targets CXX_EXECUTABLE_LINKER__ect_Release)"
ninja -C build $ect_targets
cp -r libpng/* build/optipng/libpng # prepare for below other programs
cp -r mozjpeg/* build/mozjpeg
cd ..
# ========== webp
cd webp
rm -rf build
cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \
-DZLIB_LIBRARY=$(realpath ../ect/build/zlib/libzlib.a) -DZLIB_INCLUDE_DIR=$(realpath ../ect/zlib) \
-DPNG_LIBRARY=$(realpath ../ect/build/optipng/libpng/libpng.a) -DPNG_PNG_INCLUDE_DIR=$(realpath ../ect/build/optipng/libpng) \
-DJPEG_LIBRARY=$(realpath ../ect/build/mozjpeg/libjpeg.a) -DJPEG_INCLUDE_DIR=$(realpath ../ect/build/mozjpeg) \
-DWEBP_USE_THREAD=OFF -DWEBP_UNICODE=OFF
webp_targets=""
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__webpinfo_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__cwebp_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__dwebp_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__gif2webp_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__img2webp_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__webpmux_Release)"
ninja -C build $webp_targets
cd ..
# ========== jpegxl
cd jpegxl
rm -rf build
CXXFLAGS="$CXXFLAGS"' -DHWY_DISABLED_TARGETS="(HWY_SSE2|HWY_SSSE3|HWY_SSE4|HWY_AVX3|HWY_AVX3_SPR|HWY_AVX3_ZEN4)" -DHWY_BASELINE_TARGETS=HWY_AVX2 -DHWY_COMPILE_ONLY_STATIC=ON' \
cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \
-DZLIB_LIBRARY=$(realpath ../ect/build/zlib/libzlib.a) -DZLIB_INCLUDE_DIR=$(realpath ../ect/zlib) \
-DPNG_LIBRARY=$(realpath ../ect/build/optipng/libpng/libpng.a) -DPNG_PNG_INCLUDE_DIR=$(realpath ../ect/build/optipng/libpng) \
-DJPEG_LIBRARY=$(realpath ../ect/build/mozjpeg/libjpeg.a) -DJPEG_INCLUDE_DIR=$(realpath ../ect/build/mozjpeg) \
-DBUILD_TESTING=OFF -DJPEGXL_STATIC=ON -DJPEGXL_ENABLE_BENCHMARK=OFF -DJPEGXL_ENABLE_DOXYGEN=OFF -DJPEGXL_ENABLE_MANPAGES=OFF -DJPEGXL_ENABLE_JNI=OFF -DJPEGXL_ENABLE_SJPEG=OFF -DJPEGXL_ENABLE_OPENEXR=OFF -DJPEGXL_ENABLE_AVX512=OFF
jpegxl_targets=""
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__jxlinfo_Release)"
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__cjxl_Release)"
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__djxl_Release)"
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__cjpegli_Release)"
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__djpegli_Release)"
jpegxl_targets="$(echo $jpegxl_targets | tr " " "\n" | grep -v _nocodec)" # exclude nocodecs stub implements
ninja -C build $jpegxl_targets
cd ..
# ========== multi call
g++ $CXXFLAGS multicall.cc \
$(cd ect/build ; realpath $ect_targets) \
$(cd webp/build ; realpath $webp_targets) \
$(cd jpegxl/build ; realpath $jpegxl_targets) \
-o zcodecs
# ========== [profiling]
if [ $step = final ]; then
break # is final step, skip pgo profiling
fi
run_profiling gcc
done # ^ "for step in profile final"
strip zcodecs
zip -9 linux-x64.zip zcodecs
uname -a
ldd --version
ldd zcodecs || true
./zcodecs
- uses: softprops/action-gh-release@v2
with:
prerelease: true
body: "`github.workflow_sha = ${{ github.workflow_sha }}`"
tag_name: "${{ 'asset_zcodecs_12.0.0_' }}${{ github.run_id }}"
files: linux-x64.zip
mac-arm64:
needs: prepare
runs-on: macos-14
steps:
- uses: actions/cache/restore@v4
with:
enableCrossOsArchive: true
path: zcodecs.tar
key: "asset_zcodecs_12.0.0"
- name: Build
run: |
tar -xf zcodecs.tar --strip-components 1
source utils.sh
brew install llvm@18 nasm ninja giflib # needs to set GIF_LIBRARY and GIF_INCLUDE_DIR below
export CC="clang" CXX="clang++" PATH="/opt/homebrew/opt/llvm@18/bin:$PATH" CMAKE_LINKER_TYPE=LLD
ln -s -f /opt/homebrew/opt/llvm@18/bin/llvm-ar /opt/homebrew/opt/llvm@18/bin/ar
for step in profile final; do
if [ $step = profile ]; then
export CFLAGS="-O3 -flto -fprofile-generate"
elif [ $step = final ]; then
export CFLAGS="-O3 -flto -fprofile-use=$(pwd)/pgo-0.profdata"
fi
export CXXFLAGS="$CFLAGS"
# ========== ect
cd ect
rm -rf build
cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \
-DECT_MULTITHREADING=OFF # ect use it's custom zlib, so link to system zlib is impossible
ect_targets=""
ect_targets="$ect_targets $(ninja_targets CXX_EXECUTABLE_LINKER__ect_Release)"
ninja -C build $ect_targets
cp -r libpng/* build/optipng/libpng # prepare for below other programs
cp -r mozjpeg/* build/mozjpeg
cd ..
# ========== webp
cd webp
rm -rf build
cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \
-DZLIB_LIBRARY=$(realpath ../ect/build/zlib/libzlib.a) -DZLIB_INCLUDE_DIR=$(realpath ../ect/zlib) \
-DPNG_LIBRARY=$(realpath ../ect/build/optipng/libpng/libpng.a) -DPNG_PNG_INCLUDE_DIR=$(realpath ../ect/build/optipng/libpng) \
-DJPEG_LIBRARY=$(realpath ../ect/build/mozjpeg/libjpeg.a) -DJPEG_INCLUDE_DIR=$(realpath ../ect/build/mozjpeg) \
-DGIF_LIBRARY=/opt/homebrew/opt/giflib/lib/libgif.a -DGIF_INCLUDE_DIR=/opt/homebrew/opt/giflib/include \
-DWEBP_USE_THREAD=OFF -DWEBP_UNICODE=OFF
webp_targets=""
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__webpinfo_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__cwebp_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__dwebp_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__gif2webp_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__img2webp_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__webpmux_Release)"
ninja -C build $webp_targets
cd ..
# ========== jpegxl
cd jpegxl
rm -rf build
CXXFLAGS="$CXXFLAGS"' -DHWY_DISABLED_TARGETS="(HWY_NEON_BF16|HWY_SVE|HWY_SVE2|HWY_SVE_256|HWY_SVE2_128)" -DHWY_BASELINE_TARGETS=HWY_NEON -DHWY_COMPILE_ONLY_STATIC=ON' \
cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \
-DZLIB_LIBRARY=$(realpath ../ect/build/zlib/libzlib.a) -DZLIB_INCLUDE_DIR=$(realpath ../ect/zlib) \
-DPNG_LIBRARY=$(realpath ../ect/build/optipng/libpng/libpng.a) -DPNG_PNG_INCLUDE_DIR=$(realpath ../ect/build/optipng/libpng) \
-DJPEG_LIBRARY=$(realpath ../ect/build/mozjpeg/libjpeg.a) -DJPEG_INCLUDE_DIR=$(realpath ../ect/build/mozjpeg) \
-DGIF_LIBRARY=/opt/homebrew/opt/giflib/lib/libgif.a -DGIF_INCLUDE_DIR=/opt/homebrew/opt/giflib/include \
-DBUILD_TESTING=OFF -DJPEGXL_STATIC=ON -DJPEGXL_ENABLE_BENCHMARK=OFF -DJPEGXL_ENABLE_DOXYGEN=OFF -DJPEGXL_ENABLE_MANPAGES=OFF -DJPEGXL_ENABLE_JNI=OFF -DJPEGXL_ENABLE_SJPEG=OFF -DJPEGXL_ENABLE_OPENEXR=OFF
jpegxl_targets=""
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__jxlinfo_Release)"
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__cjxl_Release)"
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__djxl_Release)"
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__cjpegli_Release)"
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__djpegli_Release)"
jpegxl_targets="$(echo $jpegxl_targets | tr " " "\n" | grep -v _nocodec)" # exclude nocodecs stub implements
ninja -C build $jpegxl_targets
cd ..
# ========== multi call
clang++ $CXXFLAGS multicall.cc \
$(cd ect/build ; realpath $ect_targets) \
$(cd webp/build ; realpath $webp_targets) \
$(cd jpegxl/build ; realpath $jpegxl_targets) \
-o zcodecs
# ========== [profiling]
if [ $step = final ]; then
break # is final step, skip pgo profiling
fi
run_profiling llvm
done # ^ "for step in profile final"
strip zcodecs
zip -9 mac-arm64.zip zcodecs
uname -a
otool -L zcodecs || true
./zcodecs
- uses: softprops/action-gh-release@v2
with:
prerelease: true
body: "`github.workflow_sha = ${{ github.workflow_sha }}`"
tag_name: "${{ 'asset_zcodecs_12.0.0_' }}${{ github.run_id }}"
files: mac-arm64.zip
win-x64:
needs: prepare
runs-on: windows-2022
steps:
- uses: actions/cache/restore@v4
with:
enableCrossOsArchive: true
path: zcodecs.tar
key: "asset_zcodecs_12.0.0"
- name: Build
shell: C:\msys64\msys2_shell.cmd -mingw64 -defterm -here -no-start -e {0}
run: |
tar -xf zcodecs.tar --strip-components 1
source utils.sh
pacman -S --needed --noconfirm zip ninja nasm mingw-w64-x86_64-cmake mingw-w64-x86_64-gcc mingw-w64-x86_64-giflib
for step in profile final; do
export CC="gcc" CXX="g++" CFLAGS="-O3 -fomit-frame-pointer -march=x86-64-v3 -flto -fprofile-update=atomic -fprofile-dir=$(cygpath -m $(pwd)/pgo)"
if [ $step = profile ]; then
export CFLAGS="$CFLAGS -fprofile-generate"
elif [ $step = final ]; then
export CFLAGS="$CFLAGS -fprofile-use"
fi
export CXXFLAGS="$CFLAGS"
# ========== ect
cd ect
rm -rf build
cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \
-DECT_MULTITHREADING=OFF # ect use it's custom zlib, so link to system zlib is impossible
ect_targets=""
ect_targets="$ect_targets $(ninja_targets CXX_EXECUTABLE_LINKER__ect_Release)"
ninja -C build $ect_targets
cp -r libpng/* build/optipng/libpng # prepare for below other programs
cp -r mozjpeg/* build/mozjpeg
cd ..
# ========== webp
cd webp
rm -rf build
cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \
-DZLIB_LIBRARY=$(realpath ../ect/build/zlib/libzlib.a) -DZLIB_INCLUDE_DIR=$(realpath ../ect/zlib) \
-DPNG_LIBRARY=$(realpath ../ect/build/optipng/libpng/libpng.a) -DPNG_PNG_INCLUDE_DIR=$(realpath ../ect/build/optipng/libpng) \
-DJPEG_LIBRARY=$(realpath ../ect/build/mozjpeg/libjpeg.a) -DJPEG_INCLUDE_DIR=$(realpath ../ect/build/mozjpeg) \
-DWEBP_USE_THREAD=OFF -DWEBP_UNICODE=OFF
webp_targets=""
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__webpinfo_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__cwebp_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__dwebp_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__gif2webp_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__img2webp_Release)"
webp_targets="$webp_targets $(ninja_targets C_EXECUTABLE_LINKER__webpmux_Release)"
ninja -C build $webp_targets
cd ..
# ========== jpegxl
cd jpegxl
rm -rf build
CXXFLAGS="$CXXFLAGS"' -DHWY_DISABLED_TARGETS="(HWY_SSE2|HWY_SSSE3|HWY_SSE4|HWY_AVX3|HWY_AVX3_SPR|HWY_AVX3_ZEN4)" -DHWY_BASELINE_TARGETS=HWY_AVX2 -DHWY_COMPILE_ONLY_STATIC=ON' \
cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \
-DZLIB_LIBRARY=$(realpath ../ect/build/zlib/libzlib.a) -DZLIB_INCLUDE_DIR=$(realpath ../ect/zlib) \
-DPNG_LIBRARY=$(realpath ../ect/build/optipng/libpng/libpng.a) -DPNG_PNG_INCLUDE_DIR=$(realpath ../ect/build/optipng/libpng) \
-DJPEG_LIBRARY=$(realpath ../ect/build/mozjpeg/libjpeg.a) -DJPEG_INCLUDE_DIR=$(realpath ../ect/build/mozjpeg) \
-DBUILD_TESTING=OFF -DJPEGXL_STATIC=ON -DJPEGXL_ENABLE_BENCHMARK=OFF -DJPEGXL_ENABLE_DOXYGEN=OFF -DJPEGXL_ENABLE_MANPAGES=OFF -DJPEGXL_ENABLE_JNI=OFF -DJPEGXL_ENABLE_SJPEG=OFF -DJPEGXL_ENABLE_OPENEXR=OFF -DJPEGXL_ENABLE_AVX512=OFF
jpegxl_targets=""
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__jxlinfo_Release)"
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__cjxl_Release)"
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__djxl_Release)"
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__cjpegli_Release)"
jpegxl_targets="$jpegxl_targets $(ninja_targets CXX_EXECUTABLE_LINKER__djpegli_Release)"
jpegxl_targets="$(echo $jpegxl_targets | tr " " "\n" | grep -v _nocodec)" # exclude nocodecs stub implements
ninja -C build $jpegxl_targets
cd ..
# ========== multi call
g++ $CXXFLAGS multicall.cc \
$(cd ect/build ; realpath $ect_targets) \
$(cd webp/build ; realpath $webp_targets) \
$(cd jpegxl/build ; realpath $jpegxl_targets) \
-lole32 -lshlwapi -static \
-o zcodecs.exe
# ========== [profiling]
if [ $step = final ]; then
break # is final step, skip pgo profiling
fi
run_profiling gcc
done # ^ "for step in profile final"
strip zcodecs.exe
zip -9 win-x64.zip zcodecs.exe
uname -a
ldd zcodecs.exe || true
./zcodecs.exe
- uses: softprops/action-gh-release@v2
with:
prerelease: true
body: "`github.workflow_sha = ${{ github.workflow_sha }}`"
tag_name: "${{ 'asset_zcodecs_12.0.0_' }}${{ github.run_id }}"
files: win-x64.zip
# description: combine many modern codecs, into a single multi-call binary
# version number: major version plus one on every modify
# why not include gifski? it's size is huge, replaced by other formats, and it process videos acturally. but we may include pngquant later?
# * 更小的 PGO 样本
# * 使用 BOLT https://github.com/llvm/llvm-project/blob/main/bolt/README.md
# ~/misc/apps/hyperfine -r 3 -s "rm -rf pgo_res ; cp -r pgo_res_bak pgo_res" \
# "./zcodecs8 cwebp -m 6 pgo_res/2.png -o pgo_res/3.png.webp" \
# "./zcodecs9 cwebp -m 6 pgo_res/2.png -o pgo_res/3.png.webp" \
# "./zcodecs0 cwebp -m 6 pgo_res/2.png -o pgo_res/3.png.webp" \
# "./zcodecs1 cwebp -m 6 pgo_res/2.png -o pgo_res/3.png.webp" \
# "./zcodecs95 cwebp -m 6 pgo_res/2.png -o pgo_res/3.png.webp" \
# ;echo finished # 0=lto-v3, 1=none
# docker run -it --rm --net host --mount type=bind,source=$(pwd)/bind,destination=/bind zcodecs /bin/bash
# llvm clang has an regression bug on x86 # -O3 -fomit-frame-pointer -march=x86-64-v3 # no_flags="-mno-aes -mno-avx -mno-avx2 -mno-f16c -mno-fma4 -mno-mmx -mno-popcnt -mno-rdrnd -mno-ssse3" # no_flags="-mno-avx -mno-f16c -mno-fma4 -mno-mmx -mno-popcnt -mno-rdrnd -mno-ssse3“