From b8907f2525e3ca9c9769a2566b1713b086fe5496 Mon Sep 17 00:00:00 2001 From: DoctorNoobingstoneIPresume Date: Tue, 26 Mar 2024 10:58:36 +0200 Subject: [PATCH] wxrc: `MakePackageCPP`: Faster&leaner compilation of C++ source code generated by our resource compiler. In order for C++-compilation to run faster and consume less memory (in the presence of thousands of input resource files) (especially with g++-14), we have changed `XRC_ADD_FILE` from a function-like macro to a function which takes not `const wxString &` parameters, but instead `const wxChar *` parameters. (Details and benchmarks are in comments in this commit.) --- utils/wxrc/GenerateBigXRCFile.sh | 41 ++++++++++++++ utils/wxrc/wxrc.cpp | 94 +++++++++++++++++++++++++++++--- 2 files changed, 126 insertions(+), 9 deletions(-) create mode 100755 utils/wxrc/GenerateBigXRCFile.sh diff --git a/utils/wxrc/GenerateBigXRCFile.sh b/utils/wxrc/GenerateBigXRCFile.sh new file mode 100755 index 000000000000..5056377bf9b8 --- /dev/null +++ b/utils/wxrc/GenerateBigXRCFile.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -e -o pipefail + +# Usage example: +# +# The following command line: +# './GenerateBigXRCFile.sh' 2000 >'MyInputFileReferencingManyResources.xrc' +# will generate an XRC file referencing 2000 small PNG images. +# +# Compiling this XRC file using `wxrc` and further compiling the generated C++ source code +# can help us tune our code and avoid slow C++ compilation process +# (when thousands of input resource files have been given as input). + +GenerateExampleImages () +{ + local folder_images='ExampleImages' + mkdir -p "${folder_images}/" + + local i + local n="${1:-10000}" + + printf '\n' + printf '\n' + for ((i = 0; i < n; ++i)); do + local pathname_image; printf -v pathname_image "%s/Example_%04Xh.png" "${folder_images}" "$((i))" + + rm -f "${pathname_image}" + printf >>"${pathname_image}" "\x89\x50\x4E\x47\x0D\x0A\x1A\x0A\x00\x00\x00\x0D\x49\x48\x44\x52" + printf >>"${pathname_image}" "\x00\x00\x00\x10\x00\x00\x00\x10\x08\x02\x00\x00\x00\x90\x91\x68" + printf >>"${pathname_image}" "\x36\x00\x00\x00\x09\x70\x48\x59\x73\x00\x00\x0E\xC4\x00\x00\x0E" + printf >>"${pathname_image}" "\xC4\x01\x95\x2B\x0E\x1B\x00\x00\x00\x1A\x49\x44\x41\x54\x28\xCF" + printf >>"${pathname_image}" "\x63\x6C\x60\xF8\xCF\x40\x0A\x60\x62\x20\x11\x8C\x6A\x18\xD5\x30" + printf >>"${pathname_image}" "\x74\x34\x00\x00\xC5\xBF\x01\x9F\x22\x91\xFF\xBD\x00\x00\x00\x00" + printf >>"${pathname_image}" "\x49\x45\x4E\x44\xAE\x42\x60\x82" + + printf ' %s\n' "$((i))" "${pathname_image}" + done + printf '\n' +} + +GenerateExampleImages "$@" diff --git a/utils/wxrc/wxrc.cpp b/utils/wxrc/wxrc.cpp index 4556496957f4..212a510e7cfd 100644 --- a/utils/wxrc/wxrc.cpp +++ b/utils/wxrc/wxrc.cpp @@ -743,13 +743,73 @@ void XmlResApp::MakePackageCPP(const wxArrayString& flist) "#include \n" "#include \n" "\n" -"#if wxCHECK_VERSION(2,8,5) && wxABI_VERSION >= 20805\n" -" #define XRC_ADD_FILE(name, data, size, mime) \\\n" -" wxMemoryFSHandler::AddFileWithMimeType(name, data, size, mime)\n" -"#else\n" -" #define XRC_ADD_FILE(name, data, size, mime) \\\n" -" wxMemoryFSHandler::AddFile(name, data, size)\n" -"#endif\n" + +// In order for C++-compilation to run faster and consume less memory +// (in the presence of thousands of input resource files) +// (especially with g++-14): +// ... Here comes a long explanation (with benchmarks too): +// +// In the C++ source code generated by the resource compiler, +// for each binary file (XRC file or file referenced from an XRC file), +// a call to `XRC_ADD_FILE` is placed in the generated file. +// +// Previously, `XRC_ADD_FILE` used to be a function-like macro +// (similar to a function which had all its calls inlined at compile-time), +// simply "forwarding" its arguments to `wxMemoryFSHandler::AddFile`. +// +// But some parameters of `wxMemoryFSHandler::AddFile` are of type `const wxString &` +// while the arguments we have are of type `const wxChar *`, +// and using the macro used to force at every "call" site +// the construction and destruction of temporary `wxString` objects. +// +// When there are thousands of such calls, the compilation of the generated C++ source code +// consumes a lot of time and memory: +// +// For 2,000 binary files: +// g++-11 (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 (`g++-11 -O2 -g`): 42.95 seconds and 1,107,296 KiB of RAM +// g++-14 (GCC) 14.0.1 20240117 (experimental) (`g++ -O2 -g`): 93.44 seconds and 1,149,840 KiB of RAM +// +// For 10,000 binary files: +// Visual C++ 2019 (version 19.29.30151 for x64) (`cl /O2 /Zi`): 46.51 seconds +// g++-11 (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 (`g++-11 -O2 -g`): 151.13 seconds and 3,265,268 KiB of RAM +// g++-14 (GCC) 14.0.1 20240117 (experimental) (`g++ -O2 -g`): 491.27 seconds and 3,489,140 KiB of RAM +// +// For 20,000 binary files: +// Visual C++ 2019 (version 19.29.30151 for x64) (`cl /O2 /Zi`): 56.46 seconds +// g++-11 (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 (`g++-11 -O2 -g`): 1,155.89 seconds and 6,217,464 KiB of RAM +// g++-14 (GCC) 14.0.1 20240117 (experimental) (`g++ -O2 -g`): 2,799.70 seconds and 6,998,852 KiB of RAM +// +// We have now changed `XRC_ADD_FILE` from a function-like macro into an actual function. +// It simply calls `wxMemoryFSHandler::AddFile`, therefore the behaviour is the same. +// But its parameters are not of type `const wxString &`, but of type `const wxChar *` instead, +// exactly matching the arguments in the generated C++ source code +// and not requiring construction and destruction of temporary `wxString` objects +// (or, at compile-time, generation of machine code which performs such constructions and destructions); +// these constructions and destructions are now centralized in the body of `XRC_ADD_FILE`. +// +// The benchmarks of compiling the generated C++ source code have significantly improved: +// +// For 2,000 binary files: +// g++-11 (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 (`g++-11 -O2 -g`): 2.22 seconds and 358,936 KiB of RAM +// g++-14 (GCC) 14.0.1 20240117 (experimental) (`g++ -O2 -g`): 4.52 seconds and 343,872 KiB of RAM +// +// For 10,000 binary files: +// Visual C++ 2019 (version 19.29.30151 for x64) (`cl /O2 /Zi`): 3.23 seconds +// g++-11 (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 (`g++-11 -O2 -g`): 13.59 seconds and 803,592 KiB of RAM +// g++-14 (GCC) 14.0.1 20240117 (experimental) (`g++ -O2 -g`): 31.80 seconds and 858,884 KiB of RAM +// +// For 20,000 binary files: +// Visual C++ 2019 (version 19.29.30151 for x64) (`cl /O2 /Zi`): 3.36 seconds +// g++-11 (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 (`g++-11 -O2 -g`): 43.47 seconds and 1,417,348 KiB of RAM +// g++-14 (GCC) 14.0.1 20240117 (experimental) (`g++ -O2 -g`): 123.11 seconds and 1,697,600 KiB of RAM +// +// (All benchmarks have been performed by x86_64 toolchains generating x86_64 machine code.) +// +// The `GenerateBigXRCFile.sh` script can be used to generate +// many small resource files and an XRC file which references them all, +// in order to conduct similar experiments/benchmarks. +// +"void XRC_ADD_FILE(const wxChar *filename, const void *binarydata, size_t size, const wxChar *mimetype);\n" "\n"); file.Write("namespace\n{\n"); @@ -806,8 +866,24 @@ void XmlResApp::MakePackageCPP(const wxArrayString& flist) } file.Write("}\n"); - - + file.Write( + "\n" + "void XRC_ADD_FILE(const wxChar *filename, const void *binarydata, size_t size, const wxChar *mimetype)\n" + "{\n" + // It may appear as though this is a simply-forwarding function, + // but in fact it converts several arguments from `const wxChar *` to `const wxString &`, + // i.e. it constructs and destroys several temporary `wxString` objects, + // and therefore we centralize these operations here as opposed to inlining them at all call sites + // (which could cost significant time and memory at compile-time and also memory at run-time + // as described in comments at the beginning of this file). + // + " #if wxCHECK_VERSION(2,8,5) && wxABI_VERSION >= 20805\n" + " return wxMemoryFSHandler::AddFileWithMimeType(filename, binarydata, size, mimetype);\n" + " #else\n" + " return wxMemoryFSHandler::AddFile(filename, binarydata, size);\n" + " #endif\n" + "}\n" + ); } void XmlResApp::GenCPPHeader()