From 798a524770f6e58fc05b5c5f0ecd35bc94a69b0b Mon Sep 17 00:00:00 2001 From: wellwei <96378453+wellwei@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:06:26 +0800 Subject: [PATCH 01/12] feat: add source-built MySQL client packages (#185) * feat: add libmysqlclient and MySQL Connector/C++ * fix(pkg): refresh compat.libmysqlclient sha after config repair The vendored libmysqlclient staging repo now defines HAVE_SYS_RESOURCE_H (plus the other POSIX macros upstream configure.cmake pairs with them) and carries Windows support sources; tag 8.4.6 was force-moved to the repaired commit, so the linux/macosx archives changed. mcpp-index still declares only linux+macosx for this package. * fix(pkg): bump compat.libmysqlclient sha (strlcat/strlcpy on Linux) Linux toolchain glibc >= 2.38 declares strlcat/strlcpy; vendored config now defines HAVE_STRLCAT/HAVE_STRLCPY there so crypt_genhash_impl.cc uses libc instead of its static fallback. Tag 8.4.6 updated again. * diag(connector): surface install-hook failures to Actions stdout The connector install hook fails with only an empty E_INTERNAL on macOS CI with no build log. Add stdout diagnostics (pcall wrapper, step values, run() error dump) to locate the failing step. Temporary diagnostic commit. * diag(connector): write hook trace to a findable build log xlings swallows install-hook stdout/log.error, so the previous stdout diagnostics never reached the Actions log. Write every step (including a descriptor-load marker) to $HOME/.mcpp/registry/data/mcpp_mysql_connector_cpp_build.log, which validate.yml's 'find ... -name mcpp_*_build.log' dump tail-prints. * diag(connector): guard diag helper for descriptor-executing lint check_mirror_urls/check_package_name execute the descriptor with only import stubbed; top-level path.join crashed lint. Make diag lazy and a no-op unless the xlings runtime's path global is present. * fix(connector): find deps in shared cache + surface hook errors Root cause (from CI diagnostics): pkginfo.install_dir('compat:openssl') returns nil when openssl was installed into the shared registry cache by the sibling libmysqlclient member instead of this member's local xpkgs, so the install hook bailed with only an empty E_INTERNAL. - declare compat:libmysqlclient/compat:openssl as xpm deps so the resolver records them for this package - add find_dep_install_dir() fallback scanning the known xpkgs roots (member-local + $MCPP_HOME/registry/data/xpkgs) - write the hook trace + failure reason to $HOME/.mcpp/registry/data/mcpp_mysql_connector_cpp_build.log, which the validate.yml find-dump tail-prints (log.error/stdout are swallowed) - macOS: sanitize PATH to /usr/bin:/bin:/usr/sbin:/sbin so a GNU libtool on PATH (conda/homebrew) cannot shadow Apple's libtool in the CMake merge * fix(connector): resolve xim cmake .app bundle, prepend system PATH CI reached CMake configure for the first time and failed with exit 127: resolve_tool fell back to bare 'cmake' because the xim cmake macOS payload is CMake.app/Contents/bin/cmake, not bin/cmake, and the previous fix replaced PATH entirely so the fallback had nothing to resolve. Handle the .app layout for an absolute path and prepend /usr/bin:/bin:/usr/sbin:/sbin to the existing PATH (system libtool wins, tools stay reachable). * fix(connector): quote PATH value in env invocation A dev-machine PATH containing spaces (e.g. 'Library/Application Support/...') split the env command line and failed CMake configure with exit 127. Wrap the prepended PATH value in single quotes (sh_quote). --- mcpp.toml | 2 + pkgs/c/compat.libmysqlclient.lua | 27 ++ pkgs/c/compat.mysql-connector-cpp.lua | 328 ++++++++++++++++++ tests/examples/libmysqlclient/mcpp.toml | 16 + .../examples/libmysqlclient/tests/client.cpp | 15 + tests/examples/mysql-connector-cpp/mcpp.toml | 17 + .../mysql-connector-cpp/tests/connector.cpp | 20 ++ 7 files changed, 425 insertions(+) create mode 100644 pkgs/c/compat.libmysqlclient.lua create mode 100644 pkgs/c/compat.mysql-connector-cpp.lua create mode 100644 tests/examples/libmysqlclient/mcpp.toml create mode 100644 tests/examples/libmysqlclient/tests/client.cpp create mode 100644 tests/examples/mysql-connector-cpp/mcpp.toml create mode 100644 tests/examples/mysql-connector-cpp/tests/connector.cpp diff --git a/mcpp.toml b/mcpp.toml index 00f6f09..4bfe121 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -44,6 +44,8 @@ members = [ "tests/examples/imgui-module", "tests/examples/imgui-window", "tests/examples/marzer.tomlplusplus", + "tests/examples/libmysqlclient", + "tests/examples/mysql-connector-cpp", "tests/examples/nlohmann.json", "tests/examples/openblas", "tests/examples/openssl", diff --git a/pkgs/c/compat.libmysqlclient.lua b/pkgs/c/compat.libmysqlclient.lua new file mode 100644 index 0000000..22411ea --- /dev/null +++ b/pkgs/c/compat.libmysqlclient.lua @@ -0,0 +1,27 @@ +-- Form A: 发布归档自带根级 mcpp.toml，消费者直接用 mcpp 构建客户端静态库。 +package = { + spec = "1", + namespace = "compat", + name = "libmysqlclient", + description = "MySQL C API client library (static, client-only source)", + licenses = {"GPL-2.0-only", "Universal-FOSS-exception-1.0"}, + repo = "https://github.com/wellwei/libmysqlclient", + type = "package", + + xpm = { + linux = { + ["8.4.6"] = { + url = "https://github.com/wellwei/libmysqlclient/archive/refs/tags/8.4.6.tar.gz", + sha256 = "27decd6716591d84a6f765a3d63fcd24031373841c684369445b83a840b38bea", + }, + }, + macosx = { + ["8.4.6"] = { + url = "https://github.com/wellwei/libmysqlclient/archive/refs/tags/8.4.6.tar.gz", + sha256 = "27decd6716591d84a6f765a3d63fcd24031373841c684369445b83a840b38bea", + }, + }, + }, + + -- 无 `mcpp` 字段：默认查找会命中归档根目录的 mcpp.toml。 +} diff --git a/pkgs/c/compat.mysql-connector-cpp.lua b/pkgs/c/compat.mysql-connector-cpp.lua new file mode 100644 index 0000000..b613bc4 --- /dev/null +++ b/pkgs/c/compat.mysql-connector-cpp.lua @@ -0,0 +1,328 @@ +-- compat.mysql-connector-cpp -- MySQL Connector/C++ X DevAPI and JDBC. +package = { + spec = "1", + namespace = "compat", + name = "mysql-connector-cpp", + description = "MySQL Connector/C++ X DevAPI and JDBC (static, source-built)", + licenses = {"GPL-2.0-only", "Universal-FOSS-exception-1.0"}, + repo = "https://github.com/mysql/mysql-connector-cpp", + type = "package", + + xpm = { + linux = { + deps = { + "compat:libmysqlclient@8.4.6", + "compat:openssl@3.5.1", + "xim:cmake@latest", + "xim:make@latest", + }, + ["26.7.0"] = { + url = "https://github.com/mysql/mysql-connector-cpp/archive/refs/tags/26.7.0.tar.gz", + sha256 = "b2299862eefc33fd71c0aac68328305671805fc955e6bd2578ef205c10f98550", + }, + }, + macosx = { + deps = { + "compat:libmysqlclient@8.4.6", + "compat:openssl@3.5.1", + "xim:cmake@latest", + }, + ["26.7.0"] = { + url = "https://github.com/mysql/mysql-connector-cpp/archive/refs/tags/26.7.0.tar.gz", + sha256 = "b2299862eefc33fd71c0aac68328305671805fc955e6bd2578ef205c10f98550", + }, + }, + }, + + mcpp = { + language = "c++23", + import_std = false, + sources = { "mcpp_mysql_connector_cpp_anchor.c" }, + include_dirs = { "include" }, + targets = { ["mysql_connector_cpp"] = { kind = "lib" } }, + deps = { + ["compat.libmysqlclient"] = "8.4.6", + ["compat.openssl"] = "3.5.1", + }, + + linux = { + ldflags = { + "-Llib", "-l:libmysqlcppconnx-static.a", + "-l:libmysqlcppconn-static.a", "-lresolv", + }, + }, + macosx = { + ldflags = { + "-Llib", "-lmysqlcppconnx-static", + "-lmysqlcppconn-static", "-lresolv", + }, + }, + }, +} + +import("xim.libxpkg.pkginfo") +import("xim.libxpkg.log") +-- Hook diagnostics: xlings swallows install-hook stdout and log.error, so a +-- failure used to surface only as an empty E_INTERNAL with no build log. This +-- hook writes a step trace + the failing reason to +-- $HOME/.mcpp/registry/data/mcpp_mysql_connector_cpp_build.log, which +-- validate.yml's `find ... -name 'mcpp_*_build.log'` dump tail-prints. It is a +-- no-op outside the xlings runtime (the `path` global is absent there), so the +-- descriptor-executing lint helpers stay green. +local hook_logf +local function hook_log(msg) + if type(path) ~= "table" then return end + if not hook_logf then + local home = os.getenv and os.getenv("HOME") or "/tmp" + hook_logf = home .. "/.mcpp/registry/data/mcpp_mysql_connector_cpp_build.log" + end + local f = io.open(hook_logf, "a") + if f then + f:write(os.date("%H:%M:%S") .. " " .. tostring(msg) .. "\n") + f:close() + end +end +hook_log("descriptor loaded") + +local function sh_quote(value) + return "'" .. tostring(value):gsub("'", "'\\''") .. "'" +end + +local function resolve_tool(dep, name, fallback) + local pkg = pkginfo.build_dep(dep) + if pkg then + local candidates = {} + if pkg.bin then candidates[#candidates + 1] = path.join(pkg.bin, name) end + if pkg.path then + -- macOS .app bundles (xim cmake ships CMake.app/Contents/bin). + candidates[#candidates + 1] = + path.join(pkg.path, name .. ".app", "Contents", "bin", name) + end + for _, cand in ipairs(candidates) do + if os.isfile(cand) then return cand end + end + end + return fallback +end + +local function tail_lines(file, count) + local ok, content = pcall(io.readfile, file) + if not ok or not content then return "" end + local lines = {} + for line in (tostring(content) .. "\n"):gmatch("(.-)\n") do + lines[#lines + 1] = line + end + return table.concat(lines, "\n", math.max(1, #lines - count + 1), #lines) +end + +local function run(step, logf, command) + -- xlings 的 os.exec 通过返回值报告非零退出；pcall 只负责捕获 Lua 错误。 + local invoked, result, reason, code = + pcall(os.exec, string.format("bash -c %s", sh_quote(command))) + if invoked and result then return true end + local err = invoked and (tostring(reason) .. " " .. tostring(code)) + or tostring(result) + local msg = "compat.mysql-connector-cpp: " .. step .. " failed (" + .. tostring(err) .. ")\n--- last 40 lines of " .. logf .. " ---\n" + .. tail_lines(logf, 40) + log.error("%s", msg) + hook_log(msg) + return false +end + +local function find_source_root() + local ifile = pkginfo.install_file() + local roots = {} + if ifile then roots[#roots + 1] = tostring(ifile):replace(".tar.gz", "") end + roots[#roots + 1] = "mysql-connector-cpp-" .. pkginfo.version() + for _, root in ipairs(roots) do + if os.isfile(path.join(root, "CMakeLists.txt")) then return root end + end +end + +local function find_mysql_source_root(prefix) + local direct = path.join(prefix, "libmysqlclient-8.4.6") + if os.isfile(path.join(direct, "mcpp.toml")) then return direct end + for _, manifest in ipairs(os.files(path.join(prefix, "**", "mcpp.toml")) or {}) do + local root = path.directory(manifest) + if os.isfile(path.join(root, "include", "mysql.h")) then return root end + end +end + +-- `pkginfo.install_dir` scans only the member-local xpkgs roots; a dependency +-- that was installed into the shared registry cache (e.g. compat.openssl built +-- by the sibling libmysqlclient member and reused here without a member-local +-- copy) is invisible to it and comes back nil. Fall back to the known xpkgs +-- roots before giving up. +local function find_dep_install_dir(dep_name, dep_version) + local dir = pkginfo.install_dir(dep_name, dep_version) + if dir then return dir end + local ns, bare = dep_name:match("^([^:]+):(.+)$") + if not ns then return nil end + local store = ns .. "-x-" .. bare + local pfx = pkginfo.install_dir() + local roots = {} + if pfx then roots[#roots + 1] = path.directory(path.directory(pfx)) end + local home = os.getenv and os.getenv("MCPP_HOME") or "" + if home == "" then home = ((os.getenv and os.getenv("HOME")) or "") .. "/.mcpp" end + roots[#roots + 1] = path.join(home, "registry/data/xpkgs") + for _, root in ipairs(roots) do + local cand = path.join(root, store, dep_version) + if os.isdir(cand) then return cand end + end + return nil +end + +local function patch_internal_compression_helper(srcroot) + local source = path.join(srcroot, "common", "session.cc") + local content = io.readfile(source) + local replacement = + "\nstatic TCPIP_options::compression_algorithm_t get_compression_algorithm(std::string alg)" + if content:find(replacement, 1, true) then return true end + + local signature = + "\nTCPIP_options::compression_algorithm_t get_compression_algorithm%(std::string alg%)" + local patched, count = content:gsub(signature, replacement) + if count ~= 1 then + log.error("compat.mysql-connector-cpp: expected one compression helper, found " + .. tostring(count)) + return false + end + io.writefile(source, patched) + return true +end + +function install() + hook_log("install() called; install_file=" .. tostring(pkginfo.install_file()) + .. " install_dir=" .. tostring(pkginfo.install_dir()) + .. " deps=" .. tostring(pkginfo.deps_list())) + local ok, result = pcall(function() + local srcroot = find_source_root() + local mysql = find_dep_install_dir("compat:libmysqlclient", "8.4.6") + local openssl = find_dep_install_dir("compat:openssl", "3.5.1") + hook_log("srcroot=" .. tostring(srcroot) .. " mysql=" .. tostring(mysql) + .. " openssl=" .. tostring(openssl)) + if not srcroot or not mysql or not openssl then + local why = "source/dependency prefix not found (srcroot=" + .. tostring(srcroot) .. " mysql=" .. tostring(mysql) + .. " openssl=" .. tostring(openssl) .. ")" + log.error("compat.mysql-connector-cpp: %s", why) + hook_log(why) + return false + end + + local mysql_src = find_mysql_source_root(mysql) + hook_log("mysql_src=" .. tostring(mysql_src)) + if not mysql_src then + log.error("compat.mysql-connector-cpp: libmysqlclient Form A source root not found") + return false + end + + local prefix = pkginfo.install_dir() + local builddir = path.join(srcroot, "mcpp-build") + local logf = path.join(prefix, "mcpp_mysql_connector_cpp_build.log") + hook_log("prefix=" .. tostring(prefix) .. " builddir=" .. tostring(builddir) + .. " logf=" .. tostring(logf)) + os.tryrm(prefix) + os.mkdir(prefix) + os.tryrm(builddir) + + -- 此 helper 只在 session.cc 内使用；内部链接避免与 MySQL 8.4 + -- libmysqlclient 中同签名的实现发生静态链接符号冲突。 + if not patch_internal_compression_helper(srcroot) then return false end + + -- Connector/C++ 的配置阶段只要求能找到一个 mysqlclient archive；真正的 + -- libmysqlclient.a 会在最终消费时由 mcpp 的依赖图链接。这里的探测库不安装。 + local mysql_probe = path.join(builddir, "mysqlclient-probe") + os.mkdir(path.join(mysql_probe, "lib")) + os.mkdir(path.join(mysql_probe, "include")) + local probe_src = path.join(mysql_probe, "probe.c") + local probe_obj = path.join(mysql_probe, "probe.o") + local probe_lib = path.join(mysql_probe, "lib", "libmysqlclient.a") + io.writefile(probe_src, "int mcpp_mysqlclient_probe(void) { return 0; }\n") + local probe_command = string.format( + "cp -R %s/. %s/ && cp %s/*.h %s/ && " + .. "cc -c %s -o %s && ar rcs %s %s", + sh_quote(path.join(mysql_src, "include")), + sh_quote(path.join(mysql_probe, "include")), + sh_quote(path.join(mysql_src, "generated")), + sh_quote(path.join(mysql_probe, "include")), + sh_quote(probe_src), sh_quote(probe_obj), sh_quote(probe_lib), sh_quote(probe_obj)) + if not run("mysqlclient probe archive", logf, + probe_command .. " >" .. sh_quote(logf) .. " 2>&1") then + return false + end + + local cmake = resolve_tool("xim:cmake", "cmake", "cmake") + local make = resolve_tool("xim:make", "make", "make") + local jobs = (os.default_njob and os.default_njob()) or 4 + local clean_env = "env -u CPPFLAGS -u CFLAGS -u CXXFLAGS -u LDFLAGS " + local compiler = "" + if os.host() == "macosx" then + -- Connector 在 project() 前启动 bootstrap CMake；必须通过环境变量 + -- 将最低系统版本同步给 bootstrap 及其后续的内置依赖构建。 + clean_env = clean_env .. "MACOSX_DEPLOYMENT_TARGET=14.0 " + -- Sanitize PATH: the connector's CMake merge step invokes Apple's + -- libtool, and a GNU libtool earlier on PATH (conda/homebrew) breaks + -- the archive merge. cmake/make come from xim by absolute path and + -- cc/c++ are pinned below, so system-only PATH is safe here. + local system_path = "/usr/bin:/bin:/usr/sbin:/sbin:" .. (os.getenv("PATH") or "") + clean_env = clean_env .. "PATH=" .. sh_quote(system_path) .. " " + compiler = "-DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_CXX_COMPILER=/usr/bin/c++ " + .. "-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 " + end + + local configure = string.format( + "%s%s -S %s -B %s -G \"Unix Makefiles\" " + .. "-DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=%s " + .. "-DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_MAKE_PROGRAM=%s " + .. "-DBUILD_STATIC=ON -DWITH_JDBC=ON -DWITH_TESTS=OFF " + .. "-DWITH_DOC=OFF -DWITH_HEADER_CHECKS=OFF " + .. "-DOPENSSL_USE_STATIC_LIBS=TRUE -DWITH_SSL=%s " + .. "-DMYSQL_INCLUDE_DIR=%s/include -DMYSQL_LIB_DIR=%s/lib " + .. "-DMYSQLCLIENT_STATIC_LINKING=ON -DMYSQLCLIENT_STATIC_BINDING=ON %s", + clean_env, sh_quote(cmake), sh_quote(srcroot), sh_quote(builddir), + sh_quote(prefix), sh_quote(make), sh_quote(openssl), + sh_quote(mysql_probe), sh_quote(mysql_probe), compiler) + if not run("CMake configure", logf, configure .. " >" .. sh_quote(logf) .. " 2>&1") then + return false + end + + local build = string.format( + "%s%s --build %s --target connector -- -j%d >>%s 2>&1", + clean_env, sh_quote(cmake), sh_quote(builddir), jobs, sh_quote(logf)) + if not run("X DevAPI build", logf, build) then return false end + + local jdbc_build = string.format( + "%s%s --build %s --target connector-jdbc -- -j%d >>%s 2>&1", + clean_env, sh_quote(cmake), sh_quote(builddir), jobs, sh_quote(logf)) + if not run("JDBC build", logf, jdbc_build) then return false end + + local install_cmd = string.format( + "%s%s --install %s --component XDevAPIDev >>%s 2>&1", + clean_env, sh_quote(cmake), sh_quote(builddir), sh_quote(logf)) + if not run("X DevAPI install", logf, install_cmd) then return false end + + local jdbc_install = string.format( + "%s%s --install %s --component JDBCDev >>%s 2>&1", + clean_env, sh_quote(cmake), sh_quote(builddir), sh_quote(logf)) + if not run("JDBC install", logf, jdbc_install) then return false end + + if not os.isfile(path.join(prefix, "include", "mysqlx", "xdevapi.h")) + or not os.isfile(path.join(prefix, "include", "mysql", "jdbc.h")) + or not os.isfile(path.join(prefix, "lib", "libmysqlcppconnx-static.a")) + or not os.isfile(path.join(prefix, "lib", "libmysqlcppconn-static.a")) then + log.error("compat.mysql-connector-cpp: incomplete X DevAPI/JDBC install") + return false + end + + io.writefile(path.join(prefix, "mcpp_mysql_connector_cpp_anchor.c"), + "int mcpp_compat_mysql_connector_cpp_anchor(void) { return 0; }\n") + end) + if not ok then + hook_log("UNCAUGHT Lua error: " .. tostring(result)) + return false + end + hook_log("install() result=" .. tostring(result)) + return result +end diff --git a/tests/examples/libmysqlclient/mcpp.toml b/tests/examples/libmysqlclient/mcpp.toml new file mode 100644 index 0000000..f473d3e --- /dev/null +++ b/tests/examples/libmysqlclient/mcpp.toml @@ -0,0 +1,16 @@ +[package] +name = "libmysqlclient-tests" +version = "0.1.0" + +# libmysqlclient 的 mcpp 源码构建目前只验证 Linux/macOS。 +[target.'cfg(linux)'.dependencies.compat] +libmysqlclient = "8.4.6" + +[target.'cfg(linux)'.build] +cxxflags = ["-DHAVE_LIBMYSQLCLIENT=1"] + +[target.'cfg(macos)'.dependencies.compat] +libmysqlclient = "8.4.6" + +[target.'cfg(macos)'.build] +cxxflags = ["-DHAVE_LIBMYSQLCLIENT=1"] diff --git a/tests/examples/libmysqlclient/tests/client.cpp b/tests/examples/libmysqlclient/tests/client.cpp new file mode 100644 index 0000000..67d212d --- /dev/null +++ b/tests/examples/libmysqlclient/tests/client.cpp @@ -0,0 +1,15 @@ +#ifdef HAVE_LIBMYSQLCLIENT +#include + +int main() { + static_assert(MYSQL_VERSION_ID == 80406); + if (mysql_get_client_version() != 80406) return 1; + + MYSQL* client = mysql_init(nullptr); + if (client == nullptr) return 2; + mysql_close(client); + return 0; +} +#else +int main() { return 0; } +#endif diff --git a/tests/examples/mysql-connector-cpp/mcpp.toml b/tests/examples/mysql-connector-cpp/mcpp.toml new file mode 100644 index 0000000..503250c --- /dev/null +++ b/tests/examples/mysql-connector-cpp/mcpp.toml @@ -0,0 +1,17 @@ +[package] +name = "mysql-connector-cpp-tests" +version = "0.1.0" + +# Connector/C++ 同时构建 X DevAPI 和 JDBC，并依赖源码构建的 +# compat.libmysqlclient/compat.openssl，因此目前只覆盖 Linux/macOS。 +[target.'cfg(linux)'.dependencies.compat] +mysql-connector-cpp = "26.7.0" + +[target.'cfg(linux)'.build] +cxxflags = ["-DHAVE_MYSQL_CONNECTOR_CPP=1"] + +[target.'cfg(macos)'.dependencies.compat] +mysql-connector-cpp = "26.7.0" + +[target.'cfg(macos)'.build] +cxxflags = ["-DHAVE_MYSQL_CONNECTOR_CPP=1"] diff --git a/tests/examples/mysql-connector-cpp/tests/connector.cpp b/tests/examples/mysql-connector-cpp/tests/connector.cpp new file mode 100644 index 0000000..2a6915b --- /dev/null +++ b/tests/examples/mysql-connector-cpp/tests/connector.cpp @@ -0,0 +1,20 @@ +#ifdef HAVE_MYSQL_CONNECTOR_CPP +#include +#include + +#include + +int main() { + // 同一个 Connector/C++ 包应同时提供 X DevAPI 和 JDBC。 + mysqlx::Value value(std::string("mcpp")); + if (value.get() != "mcpp") return 1; + + sql::mysql::MySQL_Driver* driver = sql::mysql::get_mysql_driver_instance(); + if (driver == nullptr) return 2; + if (driver->getMajorVersion() != 26) return 3; + if (driver->getMinorVersion() != 7) return 4; + return driver->getPatchVersion() == 0 ? 0 : 5; +} +#else +int main() { return 0; } +#endif From 8b2012c1de77b3a8bac76f9997b65fe590e84845 Mon Sep 17 00:00:00 2001 From: SPeak Date: Sun, 9 Aug 2026 00:22:59 +0800 Subject: [PATCH 02/12] =?UTF-8?q?fix(ci):=20=E5=88=86=E7=89=87=E6=95=B0?= =?UTF-8?q?=E8=A6=81=E6=95=B0=E8=85=BF,=E4=B8=8D=E6=98=AF=E6=95=B0=20job?= =?UTF-8?q?=20=E2=80=94=E2=80=94=20#184=20=E6=8A=8A=2065=20=E4=B8=AA?= =?UTF-8?q?=E6=88=90=E5=91=98=E8=B7=91=E6=88=90=E4=BA=86=2029=20=E4=B8=AA?= =?UTF-8?q?=20(#186)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Plan the shards` 用平台的矩阵条目数当分片数: linux:$(jq -r '[.include[]|select(.platform=="linux")]|length' ...) 在 #184 之前这两个数恰好相等,所以一直是对的。#184 给 linux 加了第二条工具链 腿之后不再相等:平台发 2 x 3 = 6 个条目,而切分仍然是 3 路,每个条目带的 shard 是 0..2。于是 plan 按 6 路切,job 只消费 0/1/2 —— 分到 3/4/5 的成员 一个都没跑。 65 个成员实际只跑了 29 个,而且是**静默**的:一个从未被分配的成员,和一个跑 过并通过的成员,在 CI 界面上长得一模一样。丢掉的里面有 ffmpeg、opencv-module 及其两个 feature 成员、catch2-v2、catch2-main、openssl —— 正是那条新腿被加进 来要测的东西。#184 全绿,但它想验证的路径一次都没执行。 改成读 `.shards`,也就是 emit() 已经写进每个条目、job 自己也在用 (matrix.shards)的那个值。plan 和消费方从此读同一个数,而不是两个碰巧相等 的数。 macos / windows 不受影响也不需要改:它们仍是单腿,条目数正好等于分片数 —— 这正是这个 bug 只咬 linux 的原因,也是它能在 review 里活下来的原因。 验证:拿 #184 那次运行的真实 matrix.json 跑 jq,linux 6 → 3,macos / windows 维持 2;plan_shards 三片合计从 29 回到 65/65,上面点名的成员全部归位。 --- .github/workflows/validate.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 27d653a..10bef6e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -519,9 +519,24 @@ jobs: { printf '{' first=1 - for spec in linux:$(jq -r '[.include[]|select(.platform=="linux")]|length' /tmp/matrix.json) \ - macos:$(jq -r '[.include[]|select(.platform=="macos")]|length' /tmp/matrix.json) \ - windows:$(jq -r '[.include[]|select(.platform=="windows")]|length' /tmp/matrix.json); do + # `.shards`, NOT the number of matrix entries. + # + # These stopped being the same number the moment linux grew a + # second toolchain leg: the platform emits 2 x 3 = 6 entries while + # the split is still 3 ways, and every entry carries shard 0..2. + # Planning 6 ways and consuming three of them dropped 36 of 65 + # members on the floor — silently, because a member that is never + # assigned is indistinguishable from one that passed. `ffmpeg`, + # `opencv-module*`, `catch2-v2` and `openssl` were among them, on + # the very run that added the leg meant to test them. + # + # `.shards` is the value emit() already wrote per entry and the + # same one the job reads for its own `matrix.shards`, so plan and + # consumer now read one number instead of two that agreed by + # accident. + for spec in linux:$(jq -r 'first(.include[]|select(.platform=="linux")).shards' /tmp/matrix.json) \ + macos:$(jq -r 'first(.include[]|select(.platform=="macos")).shards' /tmp/matrix.json) \ + windows:$(jq -r 'first(.include[]|select(.platform=="windows")).shards' /tmp/matrix.json); do p=${spec%%:*}; n=${spec##*:} [ "$first" = 1 ] || printf ',' first=0 From 6e452b9d56c8afec01e64628e15af719482b70aa Mon Sep 17 00:00:00 2001 From: SPeak Date: Sun, 9 Aug 2026 01:07:27 +0800 Subject: [PATCH 03/12] =?UTF-8?q?fix(pkg):=20catch2=20=E7=9A=84=20v2/v3=20?= =?UTF-8?q?=E5=88=A4=E5=88=AB=E4=B8=8D=E8=83=BD=E9=97=AE=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=20(#187)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `catch2_main.cpp` 用 `__has_include()` 区分两个大版本。 这个探测同样会翻系统 include 目录,而 catch_all.hpp 正是每个发行版的 catch2 包都会装的头。于是在一台装了系统 Catch2 v3 的机器上,一个 v2 消费者被回答 "你是 v3",编出 Catch::Session 那条入口,再死在链接: ld.lld: error: undefined symbol: Catch::Session::Session() >>> referenced by /usr/include/catch2/catch_session.hpp:39 和 #183 修的 compat.ffmpeg 是同一类问题:宿主机装了同名开发包,vendored 的 东西就被挤掉。gcc 不中招是因为它通过 --sysroot 进 xlings subos,那里没有 /usr/include/catch2 —— 所以这条在 CI 和默认工具链下一直是绿的。 改成探测 `catch2/catch_user_config.hpp.in`:上游把它作为 CMake 模板放在 v3 的源码树里,装的是生成后的 .hpp,模板本身从不安装。三种情形在 clang 22.1.8 和 gcc 16.1.0 上都验过: vendored v3 在 -I 上 -> v3 (两种探测一致) vendored v2 在 -I 上 -> v2 (catch_all.hpp 答 v3,即本 bug) 只有系统 v3 -> v2 (catch_all.hpp 答 v3) 四个 catch2 成员 × 两个工具链,在一台装了系统 Catch2 v3 的机器上全过 —— catch2-v2-main 此前在 llvm 下是 FAIL。 #183 的描述里把这条列为"要等 per-version build blocks(mcpp#290)才能修", 那个判断下早了:要的不是"知道自己是哪个版本",只是一个系统装不出来的探针。 mcpp#290 仍然是更干净的答案,它让这个问题整个消失,而不是换一个探针。 --- pkgs/c/compat.catch2.lua | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/pkgs/c/compat.catch2.lua b/pkgs/c/compat.catch2.lua index 7fe7d09..85f0900 100644 --- a/pkgs/c/compat.catch2.lua +++ b/pkgs/c/compat.catch2.lua @@ -29,9 +29,12 @@ -- -- Features (sources-only gate): -- `main` — compiles a GENERATED TU supplying a default entry point. It --- branches on __has_include(), which exists only in --- v3, to pick the v3 (Catch::Session) or v2 (CATCH_CONFIG_MAIN) spelling. --- Excluded by default; request `features = ["main"]`. +-- branches on __has_include() — upstream's +-- CMake template, present only in v3's SOURCE tree and never installed — to +-- pick the v3 (Catch::Session) or v2 (CATCH_CONFIG_MAIN) spelling. The probe +-- has to be source-only because __has_include searches the system dirs too; +-- see the note above the generated TU. Excluded by default; request +-- `features = ["main"]`. -- -- It deliberately does NOT point at upstream's -- src/catch2/internal/catch_main.cpp. That file is matched by the sources @@ -148,7 +151,7 @@ package = { -- and can never be used to detect v2. (On v3 it would be found -- and then fail in #include_next, since there is no upstream -- catch.hpp behind it — but __has_include never gets that far.) - -- The existing discriminator probes catch_all.hpp instead, which + -- The discriminator probes catch_user_config.hpp.in instead, which -- is unaffected. See the header comment for why per-version -- blocks (mcpp#290) are the real answer here. ["mcpp_generated/catch2/catch.hpp"] = [==[ @@ -187,8 +190,27 @@ int mcpp_compat_catch2_anchor(void) { return 0; } ]==], -- The `main` feature's TU. See the header comment for why this is -- generated rather than upstream's catch_main.cpp. + -- The discriminator probes a file upstream ships as a CMake + -- TEMPLATE and never installs: `catch_user_config.hpp.in`. It has + -- to be something a SYSTEM Catch2 cannot supply, because + -- __has_include searches the system dirs too — and the obvious + -- probe, catch_all.hpp, is installed by every distro's catch2 + -- package. On a box with system Catch2 v3 present, a v2 consumer + -- was answered "v3", compiled the Catch::Session entry point and + -- died at link time on undefined Catch::Session::Session(). + -- + -- Verified on clang 22.1.8 and gcc 16.1.0, all three cases: + -- vendored v3 on -I -> v3 (both probes agree) + -- vendored v2 on -I -> v2 (catch_all.hpp says v3: the bug) + -- only system v3 -> v2 (catch_all.hpp says v3) + -- + -- Re-check when bumping v3: if upstream ever materialises this + -- file in-tree instead of shipping the .in, or a distro starts + -- installing the template, the probe needs another source-only + -- marker. mcpp#290's per-version build blocks retire the whole + -- question by naming the major outright. ["mcpp_generated/catch2_main.cpp"] = [==[ -#if __has_include() +#if __has_include() // Catch2 v3: the library is compiled in; just drive a session. # include int main(int argc, char* argv[]) { return Catch::Session().run(argc, argv); } From 698b95ee3410f01e4b691b019ccaa80526c3e98b Mon Sep 17 00:00:00 2001 From: wellwei <96378453+wellwei@users.noreply.github.com> Date: Sun, 9 Aug 2026 02:26:34 +0800 Subject: [PATCH 04/12] feat(pkg): add compat.hiredis@1.2.0 + compat.redis-plus-plus (1.3.3 & 1.3.13) (#188) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(pkg): add compat.hiredis@1.2.0 and compat.redis-plus-plus@1.3.13 * feat(pkg): redis-plus-plus@1.3.3 — pre-watershed structure via source-list union One descriptor now serves both sides of the 1.3.6 source-structure watershed: 1.3.13 (17 sync TUs) and 1.3.3 (15 TUs, no redis_uri.cpp/redlock.cpp, no hiredis_features.h). The union works because 1.3.3's TUs are a strict subset of 1.3.13's, so exactly two source globs match nothing there (a warning, not an error — same premise as compat.catch2's disjoint union). New member tests/examples/redis-plus-plus-v133 pins 1.3.3; both members pass the same offline RESP round-trip test under mcpp 2026.8.8.2. * fix(pkg): force-include for redis-plus-plus@1.3.3 — 'uint16_t' under vendored gcc/libstdc++ 1.3.3's utils.h declares uint16_t crc16(...) without including (upstream added it in 1.3.6+). clang's libc++ pulls it in transitively, but the linux-default (gcc, vendored libstdc++) CI leg does not: 'uint16_t' does not name a type. Package cxxflags -include cstdint fixes the package build on every leg (harmless for 1.3.13, which includes it itself), and the v133 member's [build] cxxflags covers its test TU (which includes first). Caught by CI: workspace linux default 0/1. * fix(pkg): move compat.redis-plus-plus to pkgs/c/ — is the full-name initial, not the short name The repository rule is pkgs//.lua with = initial of the FULL package name (compat.* → c, docs/repository-and-schema.md). compat.redis-plus-plus was wrongly placed in pkgs/r/ (short-name initial); the compat namespace belongs in pkgs/c/ next to compat.hiredis. Resolution is path-independent, so this is purely the directory convention — README links, the member comment and the design doc updated in lock-step. --- .../2026-08-09-add-redis-plus-plus-plan.md | 95 ++++++++++ README.md | 4 +- README.zh-CN.md | 4 +- mcpp.toml | 2 + pkgs/c/compat.hiredis.lua | 99 ++++++++++ pkgs/c/compat.redis-plus-plus.lua | 143 ++++++++++++++ tests/examples/redis-plus-plus-v133/mcpp.toml | 21 +++ .../redis-plus-plus-v133/tests/redis_test.cpp | 178 ++++++++++++++++++ tests/examples/redis-plus-plus/mcpp.toml | 17 ++ .../redis-plus-plus/tests/redis_test.cpp | 178 ++++++++++++++++++ 10 files changed, 737 insertions(+), 4 deletions(-) create mode 100644 .agents/docs/2026-08-09-add-redis-plus-plus-plan.md create mode 100644 pkgs/c/compat.hiredis.lua create mode 100644 pkgs/c/compat.redis-plus-plus.lua create mode 100644 tests/examples/redis-plus-plus-v133/mcpp.toml create mode 100644 tests/examples/redis-plus-plus-v133/tests/redis_test.cpp create mode 100644 tests/examples/redis-plus-plus/mcpp.toml create mode 100644 tests/examples/redis-plus-plus/tests/redis_test.cpp diff --git a/.agents/docs/2026-08-09-add-redis-plus-plus-plan.md b/.agents/docs/2026-08-09-add-redis-plus-plus-plan.md new file mode 100644 index 0000000..ca8fcd9 --- /dev/null +++ b/.agents/docs/2026-08-09-add-redis-plus-plus-plan.md @@ -0,0 +1,95 @@ +# 新增 redis-plus-plus + hiredis 收录(compat 源码形态,双包) + +**日期**: 2026-08-09 +**本仓**: `mcpplibs/mcpp-index` +**目标**: +1. 收录 [`sewenew/redis-plus-plus`](https://github.com/sewenew/redis-plus-plus) **1.3.13** —— C++ Redis 客户端(同步 API), + 文件 `pkgs/c/compat.redis-plus-plus.lua`。 +2. 收录其依赖 [`redis/hiredis`](https://github.com/redis/hiredis) **1.2.0** —— C 客户端,文件 `pkgs/c/compat.hiredis.lua`。 +3. 形态:两者均为 **Form A 源码 compat**;redis-plus-plus 额外带一个 Shape-E 特征 + (`hiredis_features.h` 用 `generated_files` 快照)。 +4. `tests/examples/redis-plus-plus/` 最小工程:离线迷你 RESP server 断言 PING/SET 全链路。 + +版本选型原则:**不追最新,选经典、泛用、下游依赖面最广的稳定版**。后续再按 xpm 追加多版本。 + +--- + +## 1. 版本选型(经典优先) + +| 包 | 选定 | 依据 | +|---|---|---| +| hiredis | **1.2.0**(2023-06) | Debian 12 / Ubuntu 24.04 稳定版即 1.2.0;vcpkg 长期默认;conan 收录;含 `redisEnableKeepAliveWithInterval` | +| redis-plus-plus | **1.3.13**(2024-10) | 2025 快速迭代前的长期稳定版,vcpkg/conan 长期默认;API 与最新 1.3.15 一致;源码结构与 1.3.15 **逐字节一致**(`.cpp` 集合 diff 为空) | + +SHA-256(均两次独立下载一致,无归档漂移): +- hiredis 1.2.0:`82ad632d31ee05da13b537c124f819eb88e18851d9cb0c30ae0552084811588c` +- redis-plus-plus 1.3.13:`678a61898ed72f0c692102c7ce103a1bcae1e6ff85a4ad03e6002c1ba8fe1e08` + +## 2. 形态判定 + +- **hiredis → Form A(C 源码 compat)**:7 个 `.c`(上游 CMake `hiredis_sources` 原样)、头平铺、C99、无生成文件。 + 唯一特殊点:上游安装布局是 `/hiredis/hiredis.h`,tarball 却是平铺 → 用 `generated_files` 写两个 + **薄包装头**(`mcpp_generated/include/hiredis/hiredis.h` = `#include `),先例:compat.opengl 的 + `GL/gl.h`、compat.glx-headers 的 `X11/Xpoll.h`。必须是**尖括号**形式(跳过包装头自身目录,落到真实头); + `"..."` 形式会在包装目录里自包含。 +- **redis-plus-plus → Form A(C++ 源码 compat)+ Shape-E 特征**:17 个同步 TU(上游 CMake 核心清单 + + `patterns/redlock.cpp`);`async_*.cpp`/`event_loop.cpp`/`tls/…` 不编。CMake 唯一会生成的头 + `hiredis_features.h`(从 `.h.in` configure_file 产出,`connection.h` 会包含)用 `generated_files` 快照: + ```lua + ["mcpp_generated/sw/redis++/hiredis_features.h"] = "#define REDIS_PLUS_PLUS_HAS_redisEnableKeepAliveWithInterval\n" + ``` + 既非 header-only(需编译),也无 `.cppm`(非 Form C),上游无 mcpp 描述符(非 Form D)。 + +include 布局(镜像上游 target_include_directories): +- hiredis:`{ "*", "mcpp_generated/include" }` +- redis-plus-plus:`{ "*/src", "*/src/sw/redis++/cxx17", "*/src/sw/redis++/no_tls", "mcpp_generated" }` + +## 3. 多版本友好性(已跨分水岭落地) + +- hiredis 全部 1.x(1.2.0/1.3.0/1.4.1 已核实)源列表与布局一致 → 加版本 = xpm 加一行,`mcpp` 块不动。 +- redis-plus-plus **1.3.6+** 源列表与结构一致(1.3.13 vs 1.3.15 diff 为空)→ 加 1.3.14/1.3.15 = xpm 加一行。 +- **1.3.6 之前是分水岭,现已用「并集源列表」支持**:1.3.3 的同步核心是 15 个 TU(有 `shards.cpp`,缺 + `redis_uri.cpp` 与 `patterns/redlock.cpp`),且无 `hiredis_features.h`(生成的快照头只是不被包含)、 + `no_tls/tls.h` 是平铺的(`#include "tls.h"`,经 `*/src/sw/redis++/no_tls` include dir 命中)。因为 1.3.3 的 + TU 是 1.3.13 17-TU 列表的**严格子集**,同一份 `sources` 对两个版本都成立:1.3.3 上恰好两个 glob 零命中 + (警告而非错误)—— 与 compat.catch2 的「不相交并集」同款前提,这里更简单。复核新版本时须重查该子集关系 + (长期解:per-version build blocks,mcpp-community/mcpp#290)。 + +## 4. feature 评估 + +- **tls**:需 hiredis_ssl(`ssl.c`)+ OpenSSL;`compat.openssl` 仅 linux/macos,且 openssl 是 install()-驱动。 + v1 不做,留后续(可做成 hiredis `ssl` feature + redis-plus-plus `tls` feature)。 +- **async**:需 libuv,索引中无此包,需先加 `compat.libuv`。v1 不做。 +- **coro**:依赖 async。v1 不做。 +- 负向口径:不启用时 async/tls 符号应缺失(链接期 undefined reference)。 + +## 5. CN 镜像 + +本机无 `gtc`、无 `mcpp-res` 写权限 → 按 docs/cn-mirror.md 回退,**url 用纯字符串上游 GitHub release** +(先例:compat.spdlog、tensorvia-cpu)。后续由维护者补 `{ GLOBAL, CN }` 表(sha 不变)。 + +## 6. 验证结论(已实测) + +- 本地 `mcpp test -p redis-plus-plus`(1.3.13)与 `mcpp test -p redis-plus-plus-v133`(1.3.3) + (2026.8.8.4 与 CI 钉版 2026.8.8.2 各跑一遍,后者冷沙箱): + 均为 `test result ok. 1 passed; 0 failed`,测试输出 `OK: PING -> PONG, SET -> OK` —— 同一描述符、同一测试, + 两个分水岭两侧的版本都构建、链接、运行通过。 +- 独立 clang++ 冒烟(-std=c++17 与 c++23):17/17 TU 编译通过;静态链接 hiredis 后, + 离线 ping 连接被拒场景抛 `sw::redis::IoError`(完整覆盖 URI 解析 → hiredis 连接 → 错误映射)。 +- lint 全绿:`mcpp xpkg parse`(本地与钉版)、`check_mirror_urls.lua`、`check_package_name.lua`、 + `check_cross_package_refs.lua`、`check_platform_version_parity.lua`。 + +## 7. 注意事项 / 风险 + +- Windows 腿本机无法实证:`-DNOMINMAX`(redis-plus-plus,上游 CMake 同款)、 + `-D_CRT_SECURE_NO_WARNINGS -DWIN32_LEAN_AND_MEAN -lws2_32 -lcrypt32`(hiredis,上游 CMake 同款)按先例写入, + 待 CI 三平台验证;测试用与 websocket 成员相同的跨平台 socket 抽象。 +- 测试自建迷你 RESP server(PING→+PONG、SET→+OK),无需 redis-server 进程、无网络依赖; + 带 5s 接收超时防 CI 挂起。 +- 纯字符串 url 的 GLOBAL 拉取在 CI 冷缓存下需要 GitHub 可达(与 spdlog 等成员相同条件)。 +- 加 1.3.3 后,消费者侧有两个成员:`redis-plus-plus`(钉 1.3.13)与 `redis-plus-plus-v133`(钉 1.3.3), + 各自覆盖分水岭一侧;1.3.3 构建会产生两个零命中 glob 警告(预期,catch2 v2 同款)。 +- **CI 修复(PR #188 实测)**:linux-default(gcc/vendored libstdc++)腿报 + `'uint16_t' does not name a type` —— 1.3.3 的 `utils.h` 用 `uint16_t` 却未包含 `` + (1.3.6+ 才补)。包级 `cxxflags = { "-include", "cstdint" }` 修复包自身,v133 成员 + `[build] cxxflags` 修复测试 TU(先包含 redis++.h);对 1.3.13 无害。 diff --git a/README.md b/README.md index e4f0081..d4bb706 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,8 @@ Two kinds of packages live here: | Shape | Examples | |------|------| | Native module library (Form A) | [`mcpplibs.xpkg`](pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](pkgs/f/ffmpeg.lua) (module layer; sources compiled directly through `compat.ffmpeg`) · [`opencv`](pkgs/o/opencv.lua) (single repository: the module layer and the full OpenCV 5 source build both live in the package, and only this descriptor stays on the index side) · [`mcpplibs.grpc`](pkgs/g/grpc.lua) (gRPC 1.83.0 — the one library here that CANNOT be a compat descriptor: upstream publishes no self-contained source artifact, its tag archive carrying abseil/protobuf/re2/boringssl/zlib as empty submodule placeholders, so [grpc-m](https://github.com/mcpplibs/grpc-m)'s release tarball IS that artifact. It vendors only gRPC's own source and takes the five dependencies from this index, so a consumer that also uses protobuf links one copy rather than two) | -| C-source compat (with `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) | -| C++-source compat, one depending on the other | [`compat.abseil`](pkgs/c/compat.abseil.lua) (151 TUs; a wildcard over `absl/**` trimmed by upstream's test/benchmark naming conventions) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua) (the libprotobuf runtime, 79 TUs transcribed from upstream's own `src/file_lists.cmake`; declares `compat.abseil` as a dependency because protobuf's public headers include `absl/…`, and its `gzip` feature defines `HAVE_ZLIB` and pulls `compat.zlib`, while `upb` adds protobuf's 64-TU C runtime out of the same tarball. It also exposes **`protoc`** as a `kind = "bin"` target, so a consumer writing `tools = ["protoc"]` gets the compiler built for its own machine out of the same package it links — making a generator/runtime version mismatch inexpressible) · [`compat.re2`](pkgs/c/compat.re2.lua) (22 TUs, upstream's own `RE2_SOURCES`) | +| C-source compat (with `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) · [`compat.hiredis`](pkgs/c/compat.hiredis.lua) (the classic 1.2.0 — a 7-TU C build whose flat tarball headers get `hiredis/`-prefixed wrapper headers via `generated_files`, so consumers write `#include ` exactly like upstream's install layout) | +| C++-source compat, one depending on the other | [`compat.abseil`](pkgs/c/compat.abseil.lua) (151 TUs; a wildcard over `absl/**` trimmed by upstream's test/benchmark naming conventions) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua) (the libprotobuf runtime, 79 TUs transcribed from upstream's own `src/file_lists.cmake`; declares `compat.abseil` as a dependency because protobuf's public headers include `absl/…`, and its `gzip` feature defines `HAVE_ZLIB` and pulls `compat.zlib`, while `upb` adds protobuf's 64-TU C runtime out of the same tarball. It also exposes **`protoc`** as a `kind = "bin"` target, so a consumer writing `tools = ["protoc"]` gets the compiler built for its own machine out of the same package it links — making a generator/runtime version mismatch inexpressible) · [`compat.re2`](pkgs/c/compat.re2.lua) (22 TUs, upstream's own `RE2_SOURCES`) · [`compat.redis-plus-plus`](pkgs/c/compat.redis-plus-plus.lua) (redis++ 1.3.13 — the sync client, 17 TUs + `patterns/redlock.cpp`, depends on `compat.hiredis`; the one header CMake would generate, `hiredis_features.h`, is snapshotted via `generated_files`, and the async/TLS TUs are left out so the base build stays a two-package pair. Two versions, one on each side of the source-structure watershed, share this ONE source list: 1.3.13 (modern 17-TU layout) and 1.3.3 (pre-`redis_uri.cpp`/`redlock` 15-TU layout) — the union works because 1.3.3's TUs are a strict subset, so exactly two globs match nothing there (a warning, not an error; same trick as compat.catch2)) | | C++-source compat, zero-dep client + optional components | [`compat.websocket`](pkgs/c/compat.websocket.lua) (IXWebSocket 12.0.1 — a pure RFC 6455 client compiled from upstream's `IXWEBSOCKET_SOURCES` minus the four server TUs, so the **base build has zero external dependencies**: TLS off (the OpenSSL/MbedTLS/AppleSSL TUs aren't built) and `IXWEBSOCKET_USE_ZLIB` unset, so the gzip codec compiles to a no-op. Two optional features add on top: `server` (the four server TUs — `IXWebSocketServer`, `IXSocketServer`, `IXHttpServer`, `IXWebSocketProxyServer` — needing nothing external, and it **implies `zlib`** because upstream's server advertises permessage-deflate by default, which the transport negotiates regardless of the define) and `zlib` (deps `compat.zlib` and turns the codec into real per-message-deflate compression). The default-feature test brings its own minimal RFC 6455 echo server on loopback sockets (handshake, masking, fragmentation and close all exercised offline); a second member, `websocket-features`, runs a real `ix::WebSocketServer` and asserts the compression is observable on the wire — a 64 KiB repeated payload round-trips with `wireSize` = 80) | | header-only (with `features`) | [`compat.eigen`](pkgs/c/compat.eigen.lua) | | header-only, nothing to gate | [`compat.CLI11`](pkgs/c/compat.CLI11.lua) (a command line parser whose every definition is `CLI11_INLINE`, so the package is `*/include` plus an anchor TU. Upstream's two extras stay out: `src/Precompile.cpp` only means anything when `CLI11_COMPILE` also reaches the CONSUMER's translation units — an interface define, not a sources-only gate — and `src/modules/CLI11.cppm` is a module layer, which is a package shape of its own rather than a feature of the compat package) | diff --git a/README.zh-CN.md b/README.zh-CN.md index 3a93444..2ac39c0 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -35,8 +35,8 @@ mcpp self config --mirror CN # 切换至国内镜像,默认使用 GLOBAL 上 | 形态 | 示例 | |------|------| | 原生模块库(Form A) | [`mcpplibs.xpkg`](pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](pkgs/f/ffmpeg.lua)(模块层,源码经 `compat.ffmpeg` 直编) · [`opencv`](pkgs/o/opencv.lua)(单仓库:模块层与 OpenCV 5 全源码构建同在包内,索引侧只留本描述符) · [`mcpplibs.grpc`](pkgs/g/grpc.lua)(gRPC 1.83.0 —— 本索引里唯一**无法**做成 compat 描述符的库:上游不发布任何自包含源码产物,其 tag 归档里 abseil/protobuf/re2/boringssl/zlib 全是空 submodule 占位,因此 [grpc-m](https://github.com/mcpplibs/grpc-m) 的 release tarball 才是那个产物。它只 vendor gRPC 自己的源码,五个依赖全取自本索引,故同时直接使用 protobuf 的消费者链进去的是同一份而非两份)| -| C 源码 compat(含 `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) | -| C++ 源码 compat(彼此依赖) | [`compat.abseil`](pkgs/c/compat.abseil.lua)(151 TU;对 `absl/**` 取通配后,按上游自身的 test/benchmark 命名约定裁剪) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua)(libprotobuf 运行时,79 TU 逐条转录自上游 `src/file_lists.cmake`;因 protobuf 公开头文件 include 了 `absl/…`,故显式依赖 `compat.abseil`;`gzip` feature 定义 `HAVE_ZLIB` 并拉入 `compat.zlib`,`upb` feature 则从同一个 tarball 里再编出 protobuf 的 64 TU C 运行时;还以 `kind = "bin"` target 暴露 **`protoc`**,消费者写 `tools = ["protoc"]` 即可从「自己链接的那个包」拿到为本机构建的编译器,使生成器与运行时的版本错配无法表达) · [`compat.re2`](pkgs/c/compat.re2.lua)(22 TU,取自上游自身的 `RE2_SOURCES`) | +| C 源码 compat(含 `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) · [`compat.hiredis`](pkgs/c/compat.hiredis.lua)(经典 1.2.0 —— 7 个 C TU;tarball 平铺头经 `generated_files` 补 `hiredis/` 前缀薄包装头,消费者可写 `#include `,与上游安装布局一致) | +| C++ 源码 compat(彼此依赖) | [`compat.abseil`](pkgs/c/compat.abseil.lua)(151 TU;对 `absl/**` 取通配后,按上游自身的 test/benchmark 命名约定裁剪) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua)(libprotobuf 运行时,79 TU 逐条转录自上游 `src/file_lists.cmake`;因 protobuf 公开头文件 include 了 `absl/…`,故显式依赖 `compat.abseil`;`gzip` feature 定义 `HAVE_ZLIB` 并拉入 `compat.zlib`,`upb` feature 则从同一个 tarball 里再编出 protobuf 的 64 TU C 运行时;还以 `kind = "bin"` target 暴露 **`protoc`**,消费者写 `tools = ["protoc"]` 即可从「自己链接的那个包」拿到为本机构建的编译器,使生成器与运行时的版本错配无法表达) · [`compat.re2`](pkgs/c/compat.re2.lua)(22 TU,取自上游自身的 `RE2_SOURCES`) · [`compat.redis-plus-plus`](pkgs/c/compat.redis-plus-plus.lua)(redis++ 1.3.13 —— 同步客户端,17 TU + `patterns/redlock.cpp`,依赖 `compat.hiredis`;CMake 唯一会生成的头 `hiredis_features.h` 用 `generated_files` 快照,async/TLS TU 不收,基座保持两包成对。两个版本分处源码结构分水岭两侧,共享同一份源列表:1.3.13(现代 17-TU 布局)与 1.3.3(缺 `redis_uri.cpp`/`redlock` 的 15-TU 旧布局)—— 并集之所以成立,是因为 1.3.3 的 TU 是 1.3.13 的严格子集,恰好两个 glob 在 1.3.3 上零命中(仅警告,非错误;与 compat.catch2 同款手法)) | | C++ 源码 compat(零依赖客户端 + 可选组件) | [`compat.websocket`](pkgs/c/compat.websocket.lua)(IXWebSocket 12.0.1 —— 从上游 `IXWEBSOCKET_SOURCES` 剔掉 4 个 server TU 后直编的纯 RFC 6455 客户端,**基座零外部依赖**:TLS 关闭(OpenSSL/MbedTLS/AppleSSL 三组 TU 均不编),`IXWEBSOCKET_USE_ZLIB` 不定义(gzip codec 编译为 no-op)。两个可选 feature 在基座上叠加:`server`(4 个 server TU —— `IXWebSocketServer`/`IXSocketServer`/`IXHttpServer`/`IXWebSocketProxyServer`,零新增外部依赖,且 **implies `zlib`** —— 因为上游 server 默认就宣称 permessage-deflate,而 transport 的协商不受宏门控)与 `zlib`(依赖 `compat.zlib`,把 codec 变成真正的 permessage-deflate 压缩)。默认构建的测试自带基于 loopback 原始 socket 的最小 RFC 6455 echo server(握手/掩码/分片/关闭全部离线实测);第二个成员 `websocket-features` 跑真实的 `ix::WebSocketServer`,并断言压缩在线路上可观测 —— 64 KiB 重复载荷往返,`wireSize` = 80) | | header-only(含 `features`) | [`compat.eigen`](pkgs/c/compat.eigen.lua) | | header-only(无可门控组件) | [`compat.CLI11`](pkgs/c/compat.CLI11.lua)(命令行解析器,全部定义都是 `CLI11_INLINE`,故整包就是 `*/include` 加一个 anchor TU。上游两个额外件都不收:`src/Precompile.cpp` 只有在 `CLI11_COMPILE` 同时到达**消费者** TU 时才有意义 —— 那是 interface define,不是 sources 门控;`src/modules/CLI11.cppm` 属于模块层,是另一种包形态,而非 compat 包的 feature) | diff --git a/mcpp.toml b/mcpp.toml index 4bfe121..054476d 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -57,6 +57,8 @@ members = [ "tests/examples/opencv-module-dnn", "tests/examples/opencv-module-unifont", "tests/examples/re2", + "tests/examples/redis-plus-plus", + "tests/examples/redis-plus-plus-v133", "tests/examples/sdl2", "tests/examples/spdlog", "tests/examples/freetype", diff --git a/pkgs/c/compat.hiredis.lua b/pkgs/c/compat.hiredis.lua new file mode 100644 index 0000000..0620ea3 --- /dev/null +++ b/pkgs/c/compat.hiredis.lua @@ -0,0 +1,99 @@ +-- compat.hiredis — hiredis 1.2.0, minimal C client for Redis. +-- +-- Shape A (C-source compat), same as compat.cjson / compat.zlib: the seven +-- .c files upstream's own CMake compiles into libhiredis, listed verbatim. +-- ssl.c (hiredis_ssl, needs OpenSSL) and test.c stay out. +-- +-- INCLUDE LAYOUT. Upstream's INSTALL layout is `/hiredis/hiredis.h` +-- and redis-plus-plus's sources spell `#include `, but the +-- release tarball keeps every header FLAT at the root. mcpp include_dirs are +-- plain globs with no rename, so this package ships two thin wrapper headers +-- through generated_files that re-include the real flat headers with the +-- ANGLE-bracket form (which skips the wrapper's own directory): +-- mcpp_generated/include/hiredis/hiredis.h -> #include +-- mcpp_generated/include/hiredis/async.h -> #include +-- Same trick as compat.opengl's mcpp_generated/include/GL/gl.h and +-- compat.glx-headers' mcpp_generated/include/X11/Xpoll.h. The real headers' +-- internal `#include "read.h"` etc. are relative, so they resolve next to the +-- real file inside the wrap dir. +-- +-- VERSION. 1.2.0 is the classic, widely-shipped release (Debian 12 / Ubuntu +-- 24.04 stable carry 1.2.0; it was vcpkg's long-standing default). Every +-- hiredis 1.x release shares this exact source list and layout, so adding +-- 1.3.x/1.4.x later is just one more xpm row — the mcpp block never changes. +-- +-- No CN mirror yet: `url` is a plain string (upstream GitHub release only), +-- the documented fallback when there is no mcpp-res write access +-- (docs/cn-mirror.md; precedent: compat.spdlog). +package = { + spec = "1", + namespace = "compat", + name = "hiredis", + description = "Minimalistic C client for Redis (static, upstream 7-TU source build)", + licenses = {"BSD-3-Clause"}, + repo = "https://github.com/redis/hiredis", + type = "package", + + xpm = { + linux = { + ["1.2.0"] = { + url = "https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz", + sha256 = "82ad632d31ee05da13b537c124f819eb88e18851d9cb0c30ae0552084811588c", + }, + }, + macosx = { + ["1.2.0"] = { + url = "https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz", + sha256 = "82ad632d31ee05da13b537c124f819eb88e18851d9cb0c30ae0552084811588c", + }, + }, + windows = { + ["1.2.0"] = { + url = "https://github.com/redis/hiredis/archive/refs/tags/v1.2.0.tar.gz", + sha256 = "82ad632d31ee05da13b537c124f819eb88e18851d9cb0c30ae0552084811588c", + }, + }, + }, + + mcpp = { + language = "c++23", + import_std = false, + c_standard = "c99", -- hiredis requires C99 + + -- `*` is the wrap-dir root (where the flat headers live; consumers and + -- the wrappers below resolve `` through it), and + -- `mcpp_generated/include` carries the `hiredis/`-prefixed wrappers. + include_dirs = { "*", "mcpp_generated/include" }, + + -- Upstream CMakeLists' `hiredis_sources`, verbatim — identical across + -- every 1.x release. + sources = { + "*/alloc.c", + "*/async.c", + "*/hiredis.c", + "*/net.c", + "*/read.c", + "*/sds.c", + "*/sockcompat.c", + }, + + targets = { ["hiredis"] = { kind = "lib" } }, + deps = {}, + + -- Thin `hiredis/`-prefixed wrappers over the flat tarball headers, so + -- consumers write `#include ` exactly like + -- upstream's installed layout. Angle brackets are deliberate: they + -- skip the wrapper's own directory and land on the real headers via + -- the `*` include dir (the flat `"..."` form would self-include). + generated_files = { + ["mcpp_generated/include/hiredis/hiredis.h"] = "#pragma once\n#include \n", + ["mcpp_generated/include/hiredis/async.h"] = "#pragma once\n#include \n", + }, + + windows = { + -- Upstream CMake adds these on WIN32 (same pattern as compat.c-ares). + cflags = { "-D_CRT_SECURE_NO_WARNINGS", "-DWIN32_LEAN_AND_MEAN" }, + ldflags = { "-lws2_32", "-lcrypt32" }, -- propagated to consumers + }, + }, +} diff --git a/pkgs/c/compat.redis-plus-plus.lua b/pkgs/c/compat.redis-plus-plus.lua new file mode 100644 index 0000000..8b67f20 --- /dev/null +++ b/pkgs/c/compat.redis-plus-plus.lua @@ -0,0 +1,143 @@ +-- compat.redis-plus-plus — redis-plus-plus 1.3.13, C++ client for Redis. +-- +-- Shape A (C++-source compat) with one small Shape-E trait: the single header +-- upstream's CMake would generate (hiredis_features.h) is snapshotted through +-- generated_files — exactly like compat.curl's curl_config.h. The base build +-- is the SYNC client only (17 TUs from upstream CMake's +-- `REDIS_PLUS_PLUS_SOURCES` plus patterns/redlock.cpp); async (libuv) and TLS +-- (OpenSSL) TUs are not compiled, so this package has exactly one dependency: +-- compat.hiredis. +-- +-- INCLUDE LAYOUT mirrors upstream's own target_include_directories: +-- */src -> `#include ` +-- */src/sw/redis++/cxx17 -> `sw/redis++/cxx_utils.h` (the C++17 variant, +-- selected because mcpp compiles C++23) +-- */src/sw/redis++/no_tls -> `sw/redis++/tls.h` (upstream's no-op TLS stub) +-- mcpp_generated -> the generated hiredis_features.h +-- The compat.hiredis dependency's include dirs propagate, so +-- `` resolves through its `hiredis/` wrapper headers. +-- +-- generated_files snapshot: connection.h includes +-- `sw/redis++/hiredis_features.h`, which upstream produces via +-- configure_file(hiredis_features.h.in). hiredis 1.2.0 HAS +-- redisEnableKeepAliveWithInterval, so the define is on. +-- +-- VERSION / MULTI-VERSION UNION. Two versions today, one on each side of the +-- source-structure watershed: +-- * 1.3.13 (2024-10) — the classic stable release, long-standing vcpkg/conan +-- default before the 2025 releases; 1.3.6+ all share this exact source list +-- (diffed against 1.3.15: identical .cpp set), so 1.3.14/1.3.15 later are +-- just new xpm rows. +-- * 1.3.3 (2021-12) — the pre-watershed structure: 15 sync TUs (it lacks +-- redis_uri.cpp and patterns/redlock.cpp, both added in 1.3.6+), no +-- hiredis_features.h (connection.h never includes it, so the generated +-- header is simply unused), and its no_tls stub is FLAT (no_tls/tls.h, +-- included as "tls.h") instead of namespaced. +-- One mcpp block serves both because the two layouts are a SUBSET/SUPERSET +-- pair (same trick as compat.catch2's disjoint union, but simpler): the source +-- list below is 1.3.13's 17 TUs, and for 1.3.3 exactly two globs match nothing +-- (redis_uri.cpp, patterns/redlock.cpp) — a zero-hit glob is a warning, not an +-- error. Re-check the table when adding a version (mcpp-community/mcpp#290 is +-- the long-term per-version build blocks that removes this assumption). +-- +-- No CN mirror yet: plain-string upstream URL (see compat.hiredis). +package = { + spec = "1", + namespace = "compat", + name = "redis-plus-plus", + description = "C++ client for Redis (sync API; built from source, depends on compat.hiredis)", + licenses = {"Apache-2.0"}, + repo = "https://github.com/sewenew/redis-plus-plus", + type = "package", + + xpm = { + linux = { + ["1.3.3"] = { + url = "https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz", + sha256 = "23689059b7ba50fcd8fa673c5dbb7bdb010b896b14abad5aa80a5ec48eb64f26", + }, + ["1.3.13"] = { + url = "https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.13.tar.gz", + sha256 = "678a61898ed72f0c692102c7ce103a1bcae1e6ff85a4ad03e6002c1ba8fe1e08", + }, + }, + macosx = { + ["1.3.3"] = { + url = "https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz", + sha256 = "23689059b7ba50fcd8fa673c5dbb7bdb010b896b14abad5aa80a5ec48eb64f26", + }, + ["1.3.13"] = { + url = "https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.13.tar.gz", + sha256 = "678a61898ed72f0c692102c7ce103a1bcae1e6ff85a4ad03e6002c1ba8fe1e08", + }, + }, + windows = { + ["1.3.3"] = { + url = "https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.tar.gz", + sha256 = "23689059b7ba50fcd8fa673c5dbb7bdb010b896b14abad5aa80a5ec48eb64f26", + }, + ["1.3.13"] = { + url = "https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.13.tar.gz", + sha256 = "678a61898ed72f0c692102c7ce103a1bcae1e6ff85a4ad03e6002c1ba8fe1e08", + }, + }, + }, + + mcpp = { + language = "c++23", + import_std = false, + + include_dirs = { + "*/src", + "*/src/sw/redis++/cxx17", + "*/src/sw/redis++/no_tls", + "mcpp_generated", + }, + + -- Upstream CMake `REDIS_PLUS_PLUS_SOURCES` (sync core) + + -- patterns/redlock.cpp. async_*.cpp / event_loop.cpp / + -- tls/sw/redis++/tls.cpp are intentionally absent — they need + -- libuv / OpenSSL, which this base build does not pull in. + sources = { + "*/src/sw/redis++/command.cpp", + "*/src/sw/redis++/command_options.cpp", + "*/src/sw/redis++/connection.cpp", + "*/src/sw/redis++/connection_pool.cpp", + "*/src/sw/redis++/crc16.cpp", + "*/src/sw/redis++/errors.cpp", + "*/src/sw/redis++/pipeline.cpp", + "*/src/sw/redis++/redis.cpp", + "*/src/sw/redis++/redis_cluster.cpp", + "*/src/sw/redis++/redis_uri.cpp", + "*/src/sw/redis++/reply.cpp", + "*/src/sw/redis++/sentinel.cpp", + "*/src/sw/redis++/shards.cpp", + "*/src/sw/redis++/shards_pool.cpp", + "*/src/sw/redis++/subscriber.cpp", + "*/src/sw/redis++/transaction.cpp", + "*/src/sw/redis++/patterns/redlock.cpp", + }, + + targets = { ["redis_plus_plus"] = { kind = "lib" } }, + deps = { ["compat.hiredis"] = "1.2.0" }, + + generated_files = { + ["mcpp_generated/sw/redis++/hiredis_features.h"] = "#define REDIS_PLUS_PLUS_HAS_redisEnableKeepAliveWithInterval\n", + }, + + -- 1.3.3's utils.h declares `uint16_t crc16(...)` WITHOUT including + -- (upstream added the include in 1.3.6+). clang's libc++ + -- pulls it in transitively, but the vendored gcc/libstdc++ sysroot + -- does not, so the 1.3.3 build fails with "'uint16_t' does not name a + -- type" on the linux-default (gcc) CI leg. -include cstdint forces the + -- header into every C++ TU; harmless for 1.3.13 (which includes it + -- itself). Recipe repair with CI regression evidence (PR #188). + cxxflags = { "-include", "cstdint" }, + + windows = { + -- Upstream CMake adds NOMINMAX to its static-lib target on WIN32; + -- ws2_32 arrives through compat.hiredis' propagated ldflags. + cxxflags = { "-DNOMINMAX" }, + }, + }, +} diff --git a/tests/examples/redis-plus-plus-v133/mcpp.toml b/tests/examples/redis-plus-plus-v133/mcpp.toml new file mode 100644 index 0000000..40940f8 --- /dev/null +++ b/tests/examples/redis-plus-plus-v133/mcpp.toml @@ -0,0 +1,21 @@ +# redis-plus-plus v1.3.3 test project — the PRE-watershed structure: 15 sync +# TUs (no redis_uri.cpp / patterns/redlock.cpp, no hiredis_features.h, flat +# no_tls/tls.h). This is the member that proves the single-package union holds: +# at 1.3.3 exactly two source globs match nothing (a warning, not an error) and +# the build must still link and pass the same offline RESP assertions as the +# 1.3.13 member (tests/examples/redis-plus-plus). +# +# The `compat` index redirect is inherited from the workspace root. +# Part of the mcpp-index self-referential workspace. +[package] +name = "redis-plus-plus-v133-tests" +version = "0.1.0" + +[dependencies.compat] +redis-plus-plus = "1.3.3" + +# 1.3.3's utils.h uses uint16_t without including (fixed upstream in +# 1.3.6+); under the vendored gcc/libstdc++ sysroot the test TU (which includes +# first) needs the same force-include the package uses. +[build] +cxxflags = ["-include", "cstdint"] diff --git a/tests/examples/redis-plus-plus-v133/tests/redis_test.cpp b/tests/examples/redis-plus-plus-v133/tests/redis_test.cpp new file mode 100644 index 0000000..5683691 --- /dev/null +++ b/tests/examples/redis-plus-plus-v133/tests/redis_test.cpp @@ -0,0 +1,178 @@ +// redis_test.cpp — offline behavioral test for compat.redis-plus-plus. +// +// No redis-server binary and no network access: this test runs a minimal RESP +// server inside the process (raw loopback sockets, same platform abstraction +// as the websocket member) and drives the sw::redis::Redis sync client against +// it. That exercises the full stack the package must get right: +// * both static libs actually link (redis++ + hiredis), +// * hiredis connects and speaks the RESP protocol (PING -> +PONG), +// * redis++ formats commands (SET -> +OK) and parses replies back to C++. +#include + +#include +#include +#include +#include +#include +#include +#include + +// ── Platform socket abstraction (same as tests/examples/websocket) ───────── + +#ifdef _WIN32 +#include +#include +using SockType = SOCKET; +constexpr SockType kInvalidSocket = INVALID_SOCKET; +using AddrLenType = int; +#else +#include +#include +#include +#include +#include +using SockType = int; +constexpr SockType kInvalidSocket = -1; +using AddrLenType = socklen_t; +#endif + +void init_sockets() +{ +#ifdef _WIN32 + WSADATA wsa; + WSAStartup(MAKEWORD(2, 2), &wsa); +#endif +} + +void close_socket(SockType s) +{ +#ifdef _WIN32 + closesocket(s); +#else + ::close(s); +#endif +} + +// Safety net: any read that stalls this long makes the server thread return so +// the test fails cleanly instead of hanging the process (and, on CI, a runner). +void set_recv_timeout(SockType s, int ms) +{ +#ifdef _WIN32 + DWORD t = static_cast(ms); + setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, reinterpret_cast(&t), sizeof(t)); +#else + timeval tv{}; + tv.tv_sec = ms / 1000; + tv.tv_usec = (ms % 1000) * 1000; + setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); +#endif +} + +// ── Minimal RESP server: answers PING with +PONG, SET with +OK ───────────── + +class RespServer +{ +public: + ~RespServer() + { + if (client != kInvalidSocket) close_socket(client); + if (listener != kInvalidSocket) close_socket(listener); + if (thread.joinable()) thread.join(); + } + + bool start() + { + init_sockets(); + listener = socket(AF_INET, SOCK_STREAM, 0); + if (listener == kInvalidSocket) return false; + int one = 1; +#ifdef _WIN32 + setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, + reinterpret_cast(&one), sizeof(one)); +#else + setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); +#endif + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; // ephemeral + if (bind(listener, reinterpret_cast(&addr), sizeof(addr)) != 0) return false; + AddrLenType alen = sizeof(addr); + getsockname(listener, reinterpret_cast(&addr), &alen); + port = ntohs(addr.sin_port); + if (listen(listener, 1) != 0) return false; + thread = std::thread([this] { run(); }); + return true; + } + + int port = 0; + +private: + void run() + { + SockType s = accept(listener, nullptr, nullptr); + if (s == kInvalidSocket) return; + client = s; + set_recv_timeout(client, 5000); + + char buf[1024]; + int served = 0; + // The client keeps the connection open for the whole test; serve a + // handful of commands, then stop so the client's next read fails + // cleanly instead of hanging. + while (served < 8) { + const int n = static_cast(recv(client, buf, sizeof(buf) - 1, 0)); + if (n <= 0) break; + const std::string_view req(buf, static_cast(n)); + if (req.find("PING") != std::string_view::npos) { + static constexpr char kPong[] = "+PONG\r\n"; + send(client, kPong, static_cast(sizeof(kPong) - 1), 0); + ++served; + } else if (req.find("SET") != std::string_view::npos) { + static constexpr char kOk[] = "+OK\r\n"; + send(client, kOk, static_cast(sizeof(kOk) - 1), 0); + ++served; + } + } + } + + SockType listener = kInvalidSocket; + SockType client = kInvalidSocket; + std::thread thread; +}; + +int main() +{ + RespServer server; + if (!server.start()) { + std::cerr << "failed to start RESP server\n"; + return 1; + } + + sw::redis::ConnectionOptions opts; + opts.host = "127.0.0.1"; + opts.port = server.port; + opts.connect_timeout = std::chrono::milliseconds(3000); + opts.socket_timeout = std::chrono::milliseconds(3000); + + try { + sw::redis::Redis redis(opts); + + const std::string pong = redis.ping(); + if (pong != "PONG") { + std::cerr << "unexpected PING reply: '" << pong << "'\n"; + return 2; + } + + if (!redis.set("mcpp", "redis-plus-plus")) { + std::cerr << "SET did not return OK\n"; + return 3; + } + + std::cout << "OK: PING -> " << pong << ", SET -> OK\n"; + return 0; + } catch (const sw::redis::Error &e) { + std::cerr << "redis error: " << e.what() << "\n"; + return 4; + } +} diff --git a/tests/examples/redis-plus-plus/mcpp.toml b/tests/examples/redis-plus-plus/mcpp.toml new file mode 100644 index 0000000..e32f77a --- /dev/null +++ b/tests/examples/redis-plus-plus/mcpp.toml @@ -0,0 +1,17 @@ +# redis-plus-plus test project: consumes compat.redis-plus-plus (and through it +# compat.hiredis) and asserts behavior under `mcpp test`. Part of the +# mcpp-index self-referential workspace — the workspace-root `[indices]` +# redirect points at this repo, so the dependency resolves to the checked-in +# recipe (pkgs/c/compat.redis-plus-plus.lua). +# +# The test is entirely OFFLINE and self-contained: it runs a minimal RESP +# PING/SET server inside the test process (raw loopback sockets, same pattern +# as the websocket member) and drives the sw::redis::Redis sync client against +# it — a full protocol round trip with no redis-server binary and no network +# access. +[package] +name = "redis-plus-plus-tests" +version = "0.1.0" + +[dependencies.compat] +redis-plus-plus = "1.3.13" diff --git a/tests/examples/redis-plus-plus/tests/redis_test.cpp b/tests/examples/redis-plus-plus/tests/redis_test.cpp new file mode 100644 index 0000000..5683691 --- /dev/null +++ b/tests/examples/redis-plus-plus/tests/redis_test.cpp @@ -0,0 +1,178 @@ +// redis_test.cpp — offline behavioral test for compat.redis-plus-plus. +// +// No redis-server binary and no network access: this test runs a minimal RESP +// server inside the process (raw loopback sockets, same platform abstraction +// as the websocket member) and drives the sw::redis::Redis sync client against +// it. That exercises the full stack the package must get right: +// * both static libs actually link (redis++ + hiredis), +// * hiredis connects and speaks the RESP protocol (PING -> +PONG), +// * redis++ formats commands (SET -> +OK) and parses replies back to C++. +#include + +#include +#include +#include +#include +#include +#include +#include + +// ── Platform socket abstraction (same as tests/examples/websocket) ───────── + +#ifdef _WIN32 +#include +#include +using SockType = SOCKET; +constexpr SockType kInvalidSocket = INVALID_SOCKET; +using AddrLenType = int; +#else +#include +#include +#include +#include +#include +using SockType = int; +constexpr SockType kInvalidSocket = -1; +using AddrLenType = socklen_t; +#endif + +void init_sockets() +{ +#ifdef _WIN32 + WSADATA wsa; + WSAStartup(MAKEWORD(2, 2), &wsa); +#endif +} + +void close_socket(SockType s) +{ +#ifdef _WIN32 + closesocket(s); +#else + ::close(s); +#endif +} + +// Safety net: any read that stalls this long makes the server thread return so +// the test fails cleanly instead of hanging the process (and, on CI, a runner). +void set_recv_timeout(SockType s, int ms) +{ +#ifdef _WIN32 + DWORD t = static_cast(ms); + setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, reinterpret_cast(&t), sizeof(t)); +#else + timeval tv{}; + tv.tv_sec = ms / 1000; + tv.tv_usec = (ms % 1000) * 1000; + setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); +#endif +} + +// ── Minimal RESP server: answers PING with +PONG, SET with +OK ───────────── + +class RespServer +{ +public: + ~RespServer() + { + if (client != kInvalidSocket) close_socket(client); + if (listener != kInvalidSocket) close_socket(listener); + if (thread.joinable()) thread.join(); + } + + bool start() + { + init_sockets(); + listener = socket(AF_INET, SOCK_STREAM, 0); + if (listener == kInvalidSocket) return false; + int one = 1; +#ifdef _WIN32 + setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, + reinterpret_cast(&one), sizeof(one)); +#else + setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); +#endif + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; // ephemeral + if (bind(listener, reinterpret_cast(&addr), sizeof(addr)) != 0) return false; + AddrLenType alen = sizeof(addr); + getsockname(listener, reinterpret_cast(&addr), &alen); + port = ntohs(addr.sin_port); + if (listen(listener, 1) != 0) return false; + thread = std::thread([this] { run(); }); + return true; + } + + int port = 0; + +private: + void run() + { + SockType s = accept(listener, nullptr, nullptr); + if (s == kInvalidSocket) return; + client = s; + set_recv_timeout(client, 5000); + + char buf[1024]; + int served = 0; + // The client keeps the connection open for the whole test; serve a + // handful of commands, then stop so the client's next read fails + // cleanly instead of hanging. + while (served < 8) { + const int n = static_cast(recv(client, buf, sizeof(buf) - 1, 0)); + if (n <= 0) break; + const std::string_view req(buf, static_cast(n)); + if (req.find("PING") != std::string_view::npos) { + static constexpr char kPong[] = "+PONG\r\n"; + send(client, kPong, static_cast(sizeof(kPong) - 1), 0); + ++served; + } else if (req.find("SET") != std::string_view::npos) { + static constexpr char kOk[] = "+OK\r\n"; + send(client, kOk, static_cast(sizeof(kOk) - 1), 0); + ++served; + } + } + } + + SockType listener = kInvalidSocket; + SockType client = kInvalidSocket; + std::thread thread; +}; + +int main() +{ + RespServer server; + if (!server.start()) { + std::cerr << "failed to start RESP server\n"; + return 1; + } + + sw::redis::ConnectionOptions opts; + opts.host = "127.0.0.1"; + opts.port = server.port; + opts.connect_timeout = std::chrono::milliseconds(3000); + opts.socket_timeout = std::chrono::milliseconds(3000); + + try { + sw::redis::Redis redis(opts); + + const std::string pong = redis.ping(); + if (pong != "PONG") { + std::cerr << "unexpected PING reply: '" << pong << "'\n"; + return 2; + } + + if (!redis.set("mcpp", "redis-plus-plus")) { + std::cerr << "SET did not return OK\n"; + return 3; + } + + std::cout << "OK: PING -> " << pong << ", SET -> OK\n"; + return 0; + } catch (const sw::redis::Error &e) { + std::cerr << "redis error: " << e.what() << "\n"; + return 4; + } +} From 856ddc72858c4221d7b852c98f463a5c391ef180 Mon Sep 17 00:00:00 2001 From: SPeak Date: Sun, 9 Aug 2026 03:25:14 +0800 Subject: [PATCH 05/12] =?UTF-8?q?ci:=20=E5=88=B7=E6=96=B0=E8=AE=A1?= =?UTF-8?q?=E6=97=B6=E8=A1=A8=E5=B9=B6=E6=8A=8A=20linux=20=E5=88=86?= =?UTF-8?q?=E7=89=87=E4=B8=8A=E9=99=90=E6=8A=AC=E5=88=B0=204=20=E2=80=94?= =?UTF-8?q?=E2=80=94=203=20=E7=89=87=E5=B7=B2=E7=BB=8F=E8=A3=85=E4=B8=8D?= =?UTF-8?q?=E4=B8=8B=E4=BA=86=20(#189)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: 刷新计时表并把 linux 分片上限抬到 4 —— 3 片已经装不下了 `linux default 0/3` 在 run 31266814148 被 90 分钟上限砍掉,成员全绿,死在 cache/artifact 的 post 步骤。查下来既不是冷缓存也不是 runner 争抢,是工作量 真的超了,而**陈旧的计时表把这件事盖住了**。 ## 计时表漏了两个重家伙 `tests/member-timings.tsv` 停在 62 行 linux,而 workspace 已经有 67 个成员。 `plan_shards` 对表里没有的成员按**中位数**计价,于是: mysql-connector-cpp 实测 881s 被当成 ~60s libmysqlclient 实测 329s 被当成 ~60s 两个加起来 20 分钟的工作量,被当轻的塞进了已经扛着 grpc-codegen 的 shard 0。 这两个数是从 run 31266814148 的 `linux default 0/3` 步骤耗时里量的 —— 那一片 正好被砍,timings artifact 从没上传,而它们又正是导致溢出的成员。这个先有鸡还 是先有蛋只能手工破一次。表的其余部分来自 run 31260545520 的 member-timings artifact(顺带补上 cli11 / cmdline / llmapi,并修正 curl 26→264s、 eui-neo-sdl2 119→591s)。 ## 刷完表才看清:3 片本来就不够 刷新后 linux 总量 15891s = 265 分钟。按当前表模拟最慢分片: 3 片 98 分 ← 超 90 分钟 job 上限 4 片 74 分 5 片 60 分 `shards_for` 的公式 `secs / 4200 + 1` 对这个总量算出来就是 4,一直被 `cap 3` 压回 3。所以这不是新问题被引入,是旧上限被工作量长过去了 —— 旧表让 总量看起来只有 13570s,刚好还压得住。 ## 上限 3 -> 4 旧注释给这个上限的理由是"linux 实测并发 3,第 4 片会排队"。这个前提已经两头 失效:工作量长大了,而且自 #184 起 linux 每轮发 2 x N 个 job,任何 N 都会排队。 排队是对的取舍 —— 背靠背跑完的分片仍然完成,超过上限的分片不会。 代价:linux 每轮 8 个 job(原 6),模拟确认 `.shards` 读作 4、四片覆盖 67/67。 同时把那段容量注释改成现在为真的样子,并写明"表要保持新鲜"不是打扫卫生: 一个没计时的成员按中位数打包,一个重的新成员就会像轻的一样被塞进任何地方。 * ci: 改计时表不该触发全量运行 tests/member-timings.tsv 在 select 的 case 里一条都不匹配 —— 不是 tests/*.sh, 不是 tests/examples/*,不是 pkgs/*.lua,也不在忽略清单里 —— 于是落进 `*) full "unclassified change"`。 这张表决定活儿怎么在分片间分配,从不决定构建什么:没有任何成员的结果会因为 一个实测数字变了而改变。让它触发全量,是这份 workflow 里"花最大代价测试零 东西"的做法,而下一次真正的全量运行本来就会读到新数字。 代价写在注释里了:这样一来这个文件对 CI 不可见,坏行是静默的,而 plan_shards 对解析不了的东西按中位数计价 —— 正是刚刚撑爆一个分片的那个失效 模式。真要咬到就在 lint 里加校验。 * ci: 计时表只留每平台最重的 10 个,加测试不再需要动它 这张表之前列出每个成员,于是每加一个测试它就过期一次 —— 而这次撑爆分片的 正是"表里没有的成员按中位数计价":mysql-connector-cpp 实测 881s,被当成 60s。 但打包决策其实只由重的那几个做出。linux 上 top10 占总量的 69%,其余 57 个平 均 86s、中位 24s。把它们从表里拿掉,最慢分片一分钟都不动: 完整表 67 行,4 片 74 / 64 / 63 / 63 最慢 74 分 top10 表,4 片 74 / 73 / 60 / 58 最慢 74 分 ## 兜底价必须是固定常数,不能再取中位数 直接缩表会当场炸:表里只剩 10 个重的,中位数就变成"第 5 重的那个",在 linux 上是 875s,于是 57 个小成员每个都按 875s 计价 —— top10 表 + 中位数兜底 90 / 48 / 81 / 46 最慢 90 分 ← 正好撞 job 上限 改成固定 90s(未计时成员的实测均值 86s 取整)。这个值不吃调参:兜底价从 30s 到 300s,最慢分片始终在 72–84 分之间,全都在 90 以下。这种不敏感正是"表可以 长期不动"的依据。 ## shards_for 也得跟着改,否则片数会掉回去 它原本直接把表里的行加起来当工作量。表一缩,linux 总量从 15891s 读成 10965s → 3 片,而 3 片在这张表下最慢 107 分,直接超时。 给 plan_shards 加一个 ` 0 0` 模式返回总量,shards_for 改调它。这样 "有哪些成员"和"未计时的算多少钱"只有一份定义,而不是 yaml 和 lua 各一份等着 漂移。估算精度:linux 16095s vs 实测 15891s,+1.3%。macOS / windows 的成员比 默认价便宜,总量偏高 —— 两者本来就顶着 cap 2,而且偏多分片是安全方向。 ## timings job 同步产出 top10 否则下次刷新又胖回 197 行。 端到端复核:总量 16095s → 4 片,四片覆盖 67/67,最慢 74 分。 --- .github/workflows/validate.yml | 129 ++++++++++++------ tests/member-timings.tsv | 231 ++++++--------------------------- tests/plan_shards.lua | 65 ++++++++-- 3 files changed, 188 insertions(+), 237 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 10bef6e..14b65b2 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -380,6 +380,18 @@ jobs: # changes with it and the rule above selects the right members. # So a tools/ edit alone selects nothing rather than forcing a # full workspace rebuild. + # The timing table decides how work is DISTRIBUTED across + # shards, never what gets built — no member's result can change + # because a measured number moved. It used to fall through to + # the `*)` catch-all below and force a full run, which is the + # most expensive way in this workflow to test nothing: the next + # full run reads the new numbers anyway. + # + # Note this makes the file invisible to CI. A malformed row is + # then silent, and `plan_shards` prices anything it cannot parse + # at the median — the exact failure mode that overflowed a shard + # here. Guard it in `lint` if that ever bites. + tests/member-timings.tsv) : ;; *.md|docs/*|.agents/*|.github/*|tools/*) : ;; *) full "unclassified change: $f" ;; esac @@ -441,38 +453,69 @@ jobs: # platform and take one shard per ~70 minutes, which leaves ~20 # minutes of headroom under the 90-minute cap for a cold cache. # - # The caps are where runner concurrency comes back in: linux 3 (the - # measured concurrency — a 4th shard would queue), macOS 2, windows - # 2. Past those, more shards buy fit that is already there and pay - # another checkout + mcpp download + cache restore. + # The caps are where runner concurrency comes back in: linux 4, + # macOS 2, windows 2. Past those, more shards buy fit that is + # already there and pay another checkout + mcpp download + cache + # restore. # - # Full run, from the table: linux 13570s over 3 -> ~75min/shard - # (observed slowest 77), macOS 6922s over 2 -> ~57, windows 8043s - # over 2 -> ~67. linux is the tight one — its cap binds before the - # ~70-minute target does, so it is the first place to look if a - # cold full run starts brushing 90 again. The levers, in order: - # raise the linux cap to 4 (costs a queued runner), then the job + # linux was 3 — "the measured concurrency, a 4th shard would queue". + # That cap outlived its premise twice over: the workspace grew, and + # the second toolchain leg means linux emits 2 x N jobs, so it + # queues at any N. It is now 4, which is what the formula asks for. + # + # Full run, from the table: linux 15891s over 4 -> ~74min slowest + # shard, macOS 6922s over 2 -> ~57, windows 8043s over 2 -> ~67. + # + # linux is still the tight one, and the number to watch is the + # SLOWEST shard against the 90-minute job cap, not the average. + # Simulated on the current table: 3-way 98min, 4-way 74, 5-way 60. + # The levers, in order: raise the linux cap to 5, then the job # timeout. + # + # Keep tests/member-timings.tsv fresh — that is not housekeeping. An + # untimed member is priced at the MEDIAN, so a heavy newcomer packs + # like a trivial one. `mysql-connector-cpp` (881s) and + # `libmysqlclient` (329s) landed on shard 0 next to grpc-codegen + # exactly that way and pushed it past the cap, and the stale table + # then hid the overflow by under-counting the total. + # `plan_shards.lua 0 0` reports the platform's total cost. + # + # This used to sum the timing table's rows inline, which was the same + # answer only while the table listed every member. It now lists just + # the heavy ones, so summing rows measures a fraction of the work: + # 10965s instead of 16095s on linux, i.e. 3 shards where 4 are + # needed, i.e. a slowest shard of 107 minutes against a 90-minute + # cap. Asking plan_shards keeps ONE definition of both "which members + # are there" and "what does an untimed one cost". + # + # The estimate is tuned on linux (16095s vs 15891s measured, +1.3%). + # macOS and windows members are cheaper on average than the default, + # so their totals come out high — harmless here because both are + # already at their cap, and erring toward more shards is the safe + # direction. Revisit if either cap moves. shards_for() { # platform cap -> shard count - local secs - secs=$(XPLAT="$1" lua5.4 -e ' - local plat = os.getenv("XPLAT") - local sel, all = {}, (os.getenv("MEMBERS") == "__ALL__") - if not all then - for m in (os.getenv("MEMBERS") or ""):gmatch("%S+") do sel[m] = true end - end - local total = 0 - for line in io.lines("tests/member-timings.tsv") do - local p, m, s = line:match("^(%S+)\t(%S+)\t(%d+)$") - if p == plat and m and (all or sel[m]) then total = total + tonumber(s) end - end - print(total)') + local secs sel="" + [ "$MEMBERS" = "__ALL__" ] || sel="$MEMBERS" + secs=$(lua5.4 tests/plan_shards.lua "$1" 0 0 $sel) local n=$(( secs / 4200 + 1 )) [ "$n" -gt "$2" ] && n="$2" echo "$1 work: ${secs}s (measured) -> $n shard(s)" >&2 echo "$n" } - ln=$(shards_for linux 3) + # linux 3 -> 4. The cap was the binding constraint, not the ~70min + # target: `secs / 4200 + 1` asks for 4 on the refreshed table + # (15891s) and has been getting clamped back to 3. Simulated on that + # table, slowest shard: 3-way 98min (OVER the 90min job cap — and + # `linux default 0/3` was in fact cancelled at 90min on run + # 31266814148), 4-way 74min, 5-way 60min. 4 is the formula's own + # answer and leaves 16 minutes for a cold cache; 5 is there if that + # stops being enough. + # + # This does cross the concurrency line the old comment drew: with two + # toolchain legs linux now emits 8 jobs against a measured runner + # concurrency of 3, so shards queue. Queuing is the right trade — + # back-to-back shards still finish, a shard over the cap does not. + ln=$(shards_for linux 4) mn=$(shards_for macos 2) wn=$(shards_for windows 2) { @@ -945,25 +988,37 @@ jobs: # silently absorb a one-off slow runner. Refresh it deliberately — # download this artifact and replace tests/member-timings.tsv when # the numbers have actually moved. + # Top 10 per platform, not every member. + # + # plan_shards prices anything absent at a fixed default, and the + # packing decisions are made by the heavy members anyway — on linux + # the top 10 of 67 are 69% of the total, and dropping the other 57 + # moves the slowest shard by 0 minutes. A table that lists everyone + # goes stale the moment someone adds a test; one that lists ten only + # goes stale when the heavy set actually changes. That is the whole + # point: adding a member must not require touching this file. { - echo "# \t\t — from run ${{ github.run_id }}" - echo "# refresh: download the member-timings artifact and replace this file" + echo "# \t\t — the HEAVY members only, top 10" + echo "# per platform, from run ${{ github.run_id }}. Everything else is" + echo "# priced at plan_shards.lua's fixed default; adding a test does NOT" + echo "# require touching this file. Refresh when a member becomes heavy" + echo "# enough to enter the top 10, or one of these numbers moves." + # ONLY the default-toolchain leg feeds this table. The llvm leg + # runs the SAME members again, so globbing every leg would put two + # rows per (platform, member) into the file — `sort -u` keeps both, + # since the seconds differ. Both legs are planned from these + # numbers, so the default leg is the right single baseline. + # + # Then: heaviest first, keep 10. `sort -u` before that so a member + # reported by two shards cannot occupy two of the ten slots. for plat in linux macos windows; do - # ONLY the default-toolchain leg feeds this table. The llvm leg - # runs the SAME members again, so globbing every leg would put - # two rows per (platform, member) into the file — `sort -u` - # keeps both, since the seconds differ — and shards_for sums - # every matching row. Linux work would read as roughly double - # and its shard count would be permanently pinned at the cap. - # Both legs are planned from these numbers, so the default leg - # is the right single baseline. for f in timings/timings-$plat-default-*/timings.tsv; do [ -f "$f" ] || continue awk -F'\t' -v p="$plat" '{ printf "%s\t%s\t%s\n", p, $2, $1 }' "$f" - done + done | sort -u | sort -t"$(printf '\t')" -k3,3nr | head -10 done - } | sort -u > member-timings.tsv - echo "wrote member-timings.tsv ($(grep -vc '^#' member-timings.tsv) rows)" + } > member-timings.tsv + echo "wrote member-timings.tsv ($(grep -vc '^#' member-timings.tsv) rows, top 10/platform)" - name: Upload the timing table for the next run's sharding if: always() && hashFiles('member-timings.tsv') != '' diff --git a/tests/member-timings.tsv b/tests/member-timings.tsv index 8c844db..b193700 100644 --- a/tests/member-timings.tsv +++ b/tests/member-timings.tsv @@ -1,188 +1,43 @@ -linux abseil 111 -linux archive 49 -linux asio-module 37 -linux asio-ssl 49 -linux boost-ext.ut 30 -linux build-mcpp 28 -linux c-ares 28 -linux catch2 81 -linux catch2-main 53 -linux catch2-v2 13 -linux catch2-v2-main 15 -linux cjson 2 -linux core 67 -linux curl 26 -linux eigen 25 -linux eui-neo 151 -linux eui-neo-app-main 140 -linux eui-neo-markdown 143 -linux eui-neo-sdl2 119 -linux eui-neo-vulkan 154 -linux eui-neo-window 128 -linux ffmpeg 288 -linux ffmpeg-module 343 -linux fmtlib.fmt 5 -linux freetype 10 -linux glad 2 -linux godot-cpp 573 -linux godot-cpp-module 549 -linux godot-cpp-module-v10 392 -linux godot-cpp-v10 504 -linux grpc-codegen 3563 -linux grpc-module 1701 -linux gui-stack 81 -linux imgui 6 -linux imgui-module 91 -linux imgui-window 91 -linux libpng 6 -linux llamacpp 101 -linux llamacpp-metal 0 -linux magic_enum 4 -linux marzer.tomlplusplus 7 -linux md4c 3 -linux nlohmann.json 10 -linux openblas 0 -linux opencv-module 602 -linux opencv-module-dnn 741 -linux opencv-module-unifont 671 -linux openssl 0 -linux protobuf 259 -linux protobuf-gzip 158 -linux protobuf-protoc 923 -linux protobuf-upb 263 -linux re2 15 -linux sdl2 63 -linux spdlog 9 -linux spdlog-compiled 13 -linux tinyhttps 13 -linux tray 3 -linux vulkan 13 -linux websocket 19 -linux websocket-features 23 -linux yyjson 3 -macos abseil 75 -macos archive 49 -macos asio-module 30 -macos asio-ssl 78 -macos boost-ext.ut 27 -macos build-mcpp 33 -macos c-ares 27 -macos catch2 58 -macos catch2-main 28 -macos catch2-v2 10 -macos catch2-v2-main 9 -macos cjson 3 -macos core 58 -macos curl 103 -macos eigen 7 -macos eui-neo 52 -macos eui-neo-app-main 57 -macos eui-neo-markdown 76 -macos eui-neo-sdl2 142 -macos eui-neo-vulkan 59 -macos eui-neo-window 53 -macos ffmpeg 148 -macos ffmpeg-module 130 -macos fmtlib.fmt 4 -macos freetype 12 -macos glad 3 -macos godot-cpp 333 -macos godot-cpp-module 256 -macos godot-cpp-module-v10 215 -macos godot-cpp-v10 230 -macos grpc-codegen 1715 -macos grpc-module 880 -macos gui-stack 1 -macos imgui 4 -macos imgui-module 1 -macos imgui-window 1 -macos libpng 7 -macos llamacpp 66 -macos llamacpp-metal 75 -macos magic_enum 3 -macos marzer.tomlplusplus 5 -macos md4c 4 -macos nlohmann.json 9 -macos openblas 1 -macos opencv-module 198 -macos opencv-module-dnn 399 -macos opencv-module-unifont 304 -macos openssl 1 -macos protobuf 100 -macos protobuf-gzip 87 -macos protobuf-protoc 458 -macos protobuf-upb 143 -macos re2 8 -macos sdl2 22 -macos spdlog 5 -macos spdlog-compiled 6 -macos tinyhttps 13 -macos tray 4 -macos vulkan 7 -macos websocket 12 -macos websocket-features 12 -macos yyjson 6 -# — measured, run 31034885938 -# refresh: download the member-timings artifact from a full run and replace this file -windows abseil 133 -windows archive 76 -windows asio-module 37 -windows asio-ssl 24 -windows boost-ext.ut 40 -windows build-mcpp 27 -windows c-ares 33 -windows catch2 83 -windows catch2-main 62 -windows catch2-v2 12 -windows catch2-v2-main 12 -windows cjson 3 -windows core 98 -windows curl 31 -windows eigen 10 -windows eui-neo 93 -windows eui-neo-app-main 119 -windows eui-neo-markdown 84 -windows eui-neo-sdl2 150 -windows eui-neo-vulkan 84 -windows eui-neo-window 74 -windows ffmpeg 454 -windows ffmpeg-module 532 -windows fmtlib.fmt 5 -windows freetype 15 -windows glad 2 -windows godot-cpp 742 -windows godot-cpp-module 636 -windows godot-cpp-module-v10 684 -windows godot-cpp-v10 566 -windows grpc-codegen 31 -windows grpc-module 1 -windows gui-stack 1 -windows imgui 6 -windows imgui-module 0 -windows imgui-window 0 -windows libpng 10 -windows llamacpp 125 -windows llamacpp-metal 1 -windows magic_enum 4 -windows marzer.tomlplusplus 6 -windows md4c 3 -windows nlohmann.json 10 -windows openblas 11 -windows opencv-module 747 -windows opencv-module-dnn 1174 -windows opencv-module-unifont 1 -windows openssl 1 -windows protobuf 215 -windows protobuf-gzip 141 -windows protobuf-protoc 227 -windows protobuf-upb 251 -windows re2 15 -windows sdl2 46 -windows spdlog 6 -windows spdlog-compiled 14 -windows tinyhttps 14 -windows tray 2 -windows vulkan 7 -windows websocket 24 -windows websocket-features 26 -windows yyjson 2 +# \t\t — the HEAVY members only, top 10 per +# platform. Everything else is priced at plan_shards.lua's fixed default; +# on linux the 57 members not listed here average 86s and are 31% of the +# total, and listing them changes the slowest shard by 0 minutes. +# +# So: adding a test does NOT require touching this file. Refresh it when +# a member becomes heavy enough to enter the top 10, or when one of these +# numbers moves materially — download the member-timings artifact from a +# full run and keep the top 10 per platform. +# +# Sources: run 31260545520; libmysqlclient / mysql-connector-cpp measured +# from run 31266814148's linux default 0/3 step durations, whose shard was +# cancelled at the 90m cap before its timings artifact could upload. +linux grpc-codegen 3463 +linux grpc-module 1771 +linux protobuf-protoc 885 +linux mysql-connector-cpp 881 +linux opencv-module-dnn 875 +linux opencv-module-unifont 655 +linux opencv-module 649 +linux godot-cpp-module 627 +linux eui-neo-sdl2 591 +linux godot-cpp-module-v10 568 +macos grpc-codegen 1156 +macos grpc-module 719 +macos opencv-module-dnn 325 +macos protobuf-protoc 314 +macos godot-cpp-module 276 +macos opencv-module 223 +macos godot-cpp-v10 206 +macos godot-cpp-module-v10 195 +macos opencv-module-unifont 164 +macos eui-neo-sdl2 138 +windows opencv-module 868 +windows opencv-module-dnn 828 +windows godot-cpp-module 707 +windows ffmpeg 693 +windows godot-cpp-module-v10 657 +windows godot-cpp-v10 655 +windows ffmpeg-module 526 +windows protobuf-gzip 238 +windows protobuf-protoc 152 +windows protobuf-upb 151 diff --git a/tests/plan_shards.lua b/tests/plan_shards.lua index 59be162..6766313 100644 --- a/tests/plan_shards.lua +++ b/tests/plan_shards.lua @@ -26,9 +26,16 @@ -- among shards whose load is close, prefer the one already holding -- members with overlapping dependencies. -- --- Missing timing → the median, so a newly added member is neither assumed --- free nor assumed huge. No table at all → falls back to round-robin, which --- is worse but never wrong. +-- Missing timing → a FIXED default (see DEFAULT_SECONDS), so a newly added +-- member is neither assumed free nor assumed huge. No table at all → every +-- member prices the same, which degenerates to round-robin: worse, never +-- wrong. +-- +-- The table deliberately holds only the HEAVY members. They are where the +-- packing decisions actually get made — on linux the top 10 of 67 are 69% of +-- the total — and listing the other 57 buys a table that goes stale every +-- time someone adds a test. Verified equivalent: full table and top-10 table +-- both land the slowest linux shard at 74 minutes. -- -- Measured on the real workspace (linux, 3 shards), LPT against round-robin: -- @@ -71,24 +78,58 @@ end -- ── measured times ──────────────────────────────────────────────────────── -- Format: \t\t -local times, samples = {}, {} +local times = {} local tsv = read_file("tests/member-timings.tsv") if tsv then for line in tsv:gmatch("[^\n]+") do if not line:match("^#") then local p, m, s = line:match("^(%S+)\t(%S+)\t(%d+)") - if p == platform and m then - times[m] = tonumber(s) - samples[#samples + 1] = tonumber(s) - end + if p == platform and m then times[m] = tonumber(s) end end end end -local median = 60 -if #samples > 0 then - table.sort(samples) - median = samples[math.ceil(#samples / 2)] +-- Price for a member with no row. A FIXED constant, deliberately, and not the +-- median of whatever the table happens to hold. +-- +-- The table only carries the HEAVY members now (see its header). Deriving the +-- default from those samples would take the median OF THE HEAVYWEIGHTS — +-- measured at 875s on the linux set — and charge every small member that, +-- which packs far worse than having no table at all. Simulated on the full +-- 67-member measurement, 4 linux shards, slowest shard: +-- +-- full 67-row table 74 min +-- top-10 table, median-derived default 90 min <- at the job cap +-- top-10 table, fixed default 74 min +-- +-- 86s is the mean of the 57 untimed linux members (median 24s); 90 rounds it. +-- The choice is not delicate: across defaults from 30s to 300s the slowest +-- shard stays between 72 and 84 minutes, all under the 90-minute cap. That +-- insensitivity is the point — it is what lets the table sit untouched while +-- small members come and go. +-- +-- What it does NOT absorb is a new member that belongs in the heavy set. +-- `mysql-connector-cpp` (881s) priced at the default is 13 minutes of work +-- that the packer cannot see. Adding a heavy newcomer stays a real edit. +local DEFAULT_SECONDS = 90 +local median = DEFAULT_SECONDS -- name kept: read as "the price of unknown" + +-- ` ` of `0 0` prints this platform's TOTAL cost instead of a +-- shard's members. +-- +-- The workflow's shards_for needs exactly that number to pick a shard count, +-- and it needs it computed the same way: same member enumeration, same price +-- for an untimed member. It used to sum the table's rows directly, which was +-- equivalent only while the table listed every member. It no longer does — +-- summing 10 heavy rows and calling it the workload asks for 3 linux shards +-- where the real work needs 4, and 3 lands the slowest shard at 107 minutes +-- against a 90-minute cap. Two copies of "how much work is there" drift; this +-- is the one copy. +if shardCount == 0 then + local total = 0 + for _, m in ipairs(members) do total = total + (times[m] or DEFAULT_SECONDS) end + print(total) + return end -- ── run order: cheapest first ───────────────────────────────────────────── From 5ce7ee55f2c79281999ee2e8c1503f093add95e7 Mon Sep 17 00:00:00 2001 From: SPeak Date: Sun, 9 Aug 2026 03:25:27 +0800 Subject: [PATCH 06/12] =?UTF-8?q?ci:=20=E5=90=88=E5=85=A5=20main=20?= =?UTF-8?q?=E5=8F=AA=E8=B7=91=E5=AF=B9=E5=BA=94=E7=9A=84=E6=88=90=E5=91=98?= =?UTF-8?q?,=E5=85=A8=E9=87=8F=E9=80=80=E5=88=B0=E6=AF=8F=E5=91=A8=20(#191?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 每次合并都在重跑全部 67 个成员 × 三平台 —— 约 11 小时 runner 时间,用来复核 几分钟前那个 PR 已经绿过的同一批东西。 原因是 select 的第一行 `event != pull_request -> full`。它的前提是"push 没有 可 diff 的 base",而这个前提不成立:本仓的合并都是 squash,`github.event.before` 就是上一个 main,`before..HEAD` 复现出来的文件列表和 PR 当时看到的一模一样。 拿 698b95ee 实测,两边都选出 `redis-plus-plus redis-plus-plus-v133`,十个改动 文件逐个对得上。 ## 事件分派 pull_request 三点 diff(自己的提交对 merge base) push 两点 diff(这条分支上真正落下的东西) 其他 全量 schedule 和 workflow_dispatch 保持全量:它们表达的不是"有改动落地",而是 "把所有东西查一遍",那正是下面那张网的意义。 push 的两个兜底都倒向全量:before 是全零(建分支)或那个对象不在历史里 (force-push 把它丢了)时,没有可 diff 的东西,重跑好过猜。 ## 定时从每天改成每周 选择性测试结构上看不到的只有一件事:两个改动各自绿、落在一起互相踩 —— 两个 PR 的 diff 都没提到那个坏掉的成员,所以两个都选不中它。全量是这件事的网。 这张网每天撒一次和每七天撒一次,抓到的是同一批交互;而全量已经不便宜了(加 了第二条 linux 工具链腿之后约 11 小时)。改成周日 06:00 UTC。手动触发本来就 在,想立刻验一遍不用等。 五条路径都验过:schedule / dispatch / before 全零 / before 不在历史 → 全量; before 正常 → 选择性。 --- .github/workflows/validate.yml | 68 ++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 14b65b2..6917241 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -9,8 +9,18 @@ on: push: branches: [main] schedule: - # nightly full regression — exercises every workspace member regardless of diff - - cron: "0 6 * * *" + # WEEKLY full regression — every workspace member, regardless of diff. + # + # This is the safety net for the one thing selective testing structurally + # cannot see: two changes that are each green on their own and break each + # other once both have landed. Neither PR's diff names the member that + # breaks, so neither PR selects it. + # + # Weekly rather than nightly because a full run is no longer cheap — with + # the second linux toolchain leg it is ~11 hours of runner time — and the + # net catches the same interactions whether it is cast every day or every + # seven. Sunday 06:00 UTC. + - cron: "0 6 * * 0" workflow_dispatch: inputs: cache: @@ -326,21 +336,57 @@ jobs: # package. Map changed files → affected members and test only those: # pkgs//.lua → members whose mcpp.toml references # tests/examples//** → member - # Run the FULL workspace when the change can affect everything: - # non-PR events (push to main, the nightly cron, dispatch), this - # workflow file (it carries the mcpp version pins, so a version bump - # always re-validates every package), a non-member edit to the - # workspace manifest, or shared test scripts. Docs-only and tools/-only - # changes select nothing. + # A push to main is mapped the same way — the merge's own diff — so a + # merge costs what its PR cost. Run the FULL workspace when the change + # can affect everything: the weekly cron and manual dispatch (which mean + # "check everything" by definition), this workflow file (it carries the + # mcpp version pins, so a version bump always re-validates every + # package), a non-member edit to the workspace manifest, or shared test + # scripts. Docs-only and tools/-only changes select nothing. # Note: bash 3.2 on macOS runners — no associative arrays here. - name: Select affected workspace members id: plan shell: bash run: | full() { echo "MEMBERS=__ALL__" >> "$GITHUB_ENV"; echo "full run: $1"; exit 0; } - [ "${{ github.event_name }}" = "pull_request" ] || full "event=${{ github.event_name }}" - base="origin/${{ github.base_ref }}" - changed=$(git diff --name-only "$base"...HEAD) + + # A push to main has a diff too — it was just never asked for. + # + # This used to be `event != pull_request -> full`, so every merge + # re-tested all 67 members on all three platforms: ~11 hours of + # runner time to re-confirm what the PR had already gone green on + # minutes earlier. The premise was that a push has no base to diff + # against, and that is not true: merges here are squashes, so + # `github.event.before` is the previous main and + # `before..HEAD` reproduces exactly the file list the PR saw + # (verified on 698b95ee — same ten paths). + # + # schedule and workflow_dispatch stay full. They are not "a change + # landed", they are "check everything", which is the whole point of + # the weekly net above. + # + # Two-dot for push, three-dot for pull_request, deliberately: a PR + # wants its own commits against the merge base, while a push wants + # what actually landed on this branch. + case "${{ github.event_name }}" in + pull_request) + base="origin/${{ github.base_ref }}"; range="$base...HEAD" ;; + push) + base="${{ github.event.before }}" + # All-zero on branch creation; absent object after a + # force-push that dropped it. Either way there is nothing to + # diff against, and guessing is worse than re-testing. + case "$base" in + ""|0000000000000000000000000000000000000000) + full "push with no predecessor" ;; + esac + git cat-file -e "$base^{commit}" 2>/dev/null \ + || full "push predecessor $base not in history" + range="$base..HEAD" ;; + *) + full "event=${{ github.event_name }}" ;; + esac + changed=$(git diff --name-only $range) printf 'changed files vs %s:\n%s\n' "$base" "$changed" sel="" add() { case " $sel " in *" $1 "*) ;; *) sel="$sel $1" ;; esac; } From b86fc7c0c80a93f4ccdf797c13ec0cca2557d6eb Mon Sep 17 00:00:00 2001 From: wellwei <96378453+wellwei@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:27:36 +0800 Subject: [PATCH 07/12] =?UTF-8?q?feat(pkg):=20add=20compat.sqlite3@3.45.3?= =?UTF-8?q?=20=E2=80=94=20SQLite=20C=20amalgamation=20(most=20widely=20dep?= =?UTF-8?q?loyed=203.45.x=20line)=20(#190)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Shape A (C-source compat): compile the single sqlite3.c amalgamation into a lib, expose sqlite3.h/sqlite3ext.h via include_dirs; shell.c (interactive CLI, needs editline/readline) stays out. - Version 3.45.3 = final maintenance release of the most widely deployed SQLite series (Ubuntu 24.04 LTS ships 3.45.1; CPython 3.11/3.12/3.13 installers ship 3.45.1/3.45.3/3.45.3), chosen over newer series per user request (widely-used, not latest). - GLOBAL = sqlite.org amalgamation zip (sha256 verified twice); no CN mirror (plain-string url fallback — the GitHub sqlite/sqlite mirror does not carry the generated sqlite3.c). - Consumer member tests/examples/sqlite3 asserts sqlite3_libversion()==3.45.3 and exercises open/create/insert/prepared-query end to end; verified cold with the CI-pinned mcpp 2026.8.8.2 (test result ok, 1 passed). - Design doc covers shape decision, version rationale, multi-version support plan (one mcpp block, xpm rows only). --- .agents/docs/2026-08-09-add-sqlite3-plan.md | 114 ++++++++++++++++++ README.md | 2 +- README.zh-CN.md | 2 +- mcpp.toml | 1 + pkgs/c/compat.sqlite3.lua | 60 +++++++++ tests/examples/sqlite3/mcpp.toml | 9 ++ tests/examples/sqlite3/tests/sqlite3_test.cpp | 39 ++++++ 7 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 .agents/docs/2026-08-09-add-sqlite3-plan.md create mode 100644 pkgs/c/compat.sqlite3.lua create mode 100644 tests/examples/sqlite3/mcpp.toml create mode 100644 tests/examples/sqlite3/tests/sqlite3_test.cpp diff --git a/.agents/docs/2026-08-09-add-sqlite3-plan.md b/.agents/docs/2026-08-09-add-sqlite3-plan.md new file mode 100644 index 0000000..5f13369 --- /dev/null +++ b/.agents/docs/2026-08-09-add-sqlite3-plan.md @@ -0,0 +1,114 @@ +# 新增 SQLite 收录(compat.sqlite3,C 源码 compat) + +**日期**: 2026-08-09 +**本仓**: `mcpplibs/mcpp-index` +**参考**: PR #48(cJSON compat)、#50(eigen);skill [`add-mcpp-index-package`](../skills/add-mcpp-index-package/SKILL.md) +**目标**: 收录 SQLite 引擎本体为 `compat.sqlite3`(C 源码 compat 形态),版本 **3.45.3**,用户 +`#include ` 开箱即用;最小示例 `tests/examples/sqlite3/` 走完整冷验证。 + +--- + +## 1. 版本选择:3.45.3(部署最广,而非最新) + +用户要求"选最广泛使用的版本,不追求最新"。调研结论:3.45.x 是当前部署面最广的版本线。 + +| 平台 | SQLite 版本 | +|---|---| +| Ubuntu 24.04 LTS(支持至 2029+) | 3.45.1(安全更新在 3.45.x 线内) | +| CPython 3.11 / 3.12 / 3.13 官方安装器 | 3.45.1 / 3.45.3 / 3.45.3 | +| Debian 13(trixie)/ Debian 12(bookworm) | 3.46.1 / 3.40.1(3.45 处于两者交集) | +| Android 15+ / iOS 18+ 时代生态 | 3.45+ 被普遍视为兼容基准 | + +- **3.45.3** = 3.45.x 线最后一个维护版(含该线安全/稳定性修复),API 与 Ubuntu 24.04 LTS 的 3.45.1 一致,更成熟。 +- 未选最新 3.53.4:新版本线部署面尚未铺开,收录后对消费侧无兼容收益。 +- 后续可加新版本线:xpm 加行即可,mcpp 块不变。 + +## 2. 上游布局与哈希(已实测) + +- GLOBAL:`https://sqlite.org/2024/sqlite-amalgamation-3450300.zip`(HEAD 200 已验证;3.45.1/3.45.2/3.45.3 + 均存在,年路径为发布年)。 +- sha256(两次计算一致):`ea170e73e447703e8359308ca2e4366a3ae0c4304a8665896f068c736781c651`。 +- zip 单层 wrap `sqlite-amalgamation-3450300/`,内含 `sqlite3.c`(9.0 MB)、`shell.c`、`sqlite3.h`、 + `sqlite3ext.h`。glob `*` 吸收 wrap 层。 +- 许可:Public Domain(SPDX 无标准 id,采用 `LicenseRef-Public-Domain` 惯例)。 + +## 3. 形态判定:形态 A(C 源码 compat),无 feature + +- 只编 `sqlite3.c` 进 lib;`shell.c`(交互式 CLI)不编 —— 它依赖 editline/readline。 +- `include_dirs = {"*"}` 暴露 `sqlite3.h` / `sqlite3ext.h`。 +- amalgamation 开箱即编,无需任何 define/config;`c_standard = c11`(与 compat.cjson 对齐)。 +- **无可门控组件**:amalgamation 是单一 TU,可选功能全部由编译期 define 控制(如 + `SQLITE_OMIT_*` / `SQLITE_ENABLE_*`),而当前 mcpp feature 表只能门控 sources、不能携带 define → 不实现 + feature(同 compat.eigen 对 define 类开关的结论)。 +- **无 CN 镜像**:sqlite.org 是 amalgamation 的唯一权威来源(GitHub `sqlite/sqlite` 镜像仓**不含**生成的 + `sqlite3.c`,raw 404 实测);无 `mcpp-res` 写权限 → 采用纯字符串 url 回退(先例:compat.hiredis / + compat.spdlog / compat.redis-plus-plus)。 + +## 4. 描述符与消费者示例 + +- `pkgs/c/compat.sqlite3.lua`:三平台同 url+sha;`sources = {"*/sqlite3.c"}`;target `sqlite3`(kind lib)。 +- `tests/examples/sqlite3/`:`mcpp.toml` 依赖 `compat.sqlite3 = "3.45.3"`;`tests/sqlite3_test.cpp` 断言 + `sqlite3_libversion() == "3.45.3"`,并走 `sqlite3_open(:memory:) → sqlite3_exec 建表/插入 → 预编译语句查询` + 全链路。 +- 根 `mcpp.toml` `[workspace] members` 登记 `tests/examples/sqlite3`(CI 的成员选择与 `--all` 都读它)。 + +## 5. 本地验证(mcpp 2026.8.8.2,与 CI 同版) + +- 环境:`MCPP`/`MCPP_HOME`/`MCPP_VENDORED_XLINGS` 指向 `mcpp-2026.8.8.2-macosx-arm64` 解包根, + `MCPP_INDEX_MIRROR=GLOBAL`;`~/.mcpp/registry` 复制 release 自带 registry。 +- 结果(`$MCPP test -p sqlite3`,冷状态,自动装 llvm@20.1.7 工具链): + `sqlite3_test ... ok (0.27s)` / `test result ok. 1 passed; 0 failed; finished in 104.29s`。 +- 包从 sqlite.org 下载、`sqlite3.c` 编译、链接、运行断言全部通过。 + +## 6. lint(复现 validate.yml) + +- lua 语法 `loadfile(...,'t')` OK;`spec/name/xpm` 必填字段齐全;无前导 v。 +- `check_mirror_urls.lua` OK(纯字符串 url 不施加镜像约束)。 +- `check_package_name.lua` OK;全仓 `check_cross_package_refs.lua` OK。 + +## 7. 后续可选项(不在本 PR 范围) + +- **C++ 封装层**(SQLiteCpp / sqlite_orm):用户已评估 —— 非必要,引擎先行;若后续做,sqlite_orm 为 + header-only 且提供 MIT 双许可,适合独立 PR。 +- **CN 镜像**:获得 `mcpp-res` 写权限后,可将 url 改写为 `{ GLOBAL=…, CN=… }` 表(sha256 不变)。 +- **新版本线**:3.46+/3.50+/3.53+ 作为新 xpm 行追加。 + +## 8. 多版本支持方案(后续可实施,难度低) + +SQLite amalgamation 布局几十年来恒定(每版均为 `sqlite3.c` / `sqlite3.h` / `sqlite3ext.h` / `shell.c`, +同一个 wrap 层),因此**一个 `mcpp` 块通吃所有版本**:`sources` / `include_dirs` / `c_standard` 不变, +加版本 = 三平台各加一行 `xpm` 条目。对比本仓难例(`compat.catch2` v2/v3 形态切换、`compat.redis-plus-plus` +subset/superset 源码并集),SQLite 属最简单一档。 + +### 8.1 URL 数字编码与年路径(唯一需要小心的点) + +版本号 → 数字编码 = 大版本 1 位 + 次版本 2 位 + 补丁 2 位;URL 带**发布年**路径。已实测: + +| 版本 | URL | 备注 | +|---|---|---| +| 3.45.3 | `https://sqlite.org/2024/sqlite-amalgamation-3450300.zip` | 本 PR 收录 | +| 3.46.1 | `https://sqlite.org/2024/sqlite-amalgamation-3460100.zip` | 3.46 → `34601` | +| 3.50.0 | `https://sqlite.org/2025/sqlite-amalgamation-3500000.zip` | 3.50 → `35000`,勿写成 `350` | +| 3.53.4 | `https://sqlite.org/2026/sqlite-amalgamation-3530400.zip` | 年路径随发布年变 | + +### 8.2 加一版的操作步骤 + +1. 下载对应 zip,`tar -tzf` 确认布局仍为四件套(应无变化)。 +2. `sha256sum` 计算两次确认稳定。 +3. 描述符三平台各加一个 `["x.y.z"] = { url=…, sha256=… }` 行(mcpp 块不动)。 +4. 新成员 `tests/examples/sqlite3-/`(或复用既有成员改 pin)做冷验证。 + +### 8.3 测试与 CI 覆盖策略 + +- **每版本一个成员、断言各自 pin 的版本**(先例:`tests/examples/redis-plus-plus` 与 + `redis-plus-plus-v133` 各 pin 一版),不放松 `sqlite3_libversion()` 断言 —— 否则「解析到的确为所 pin + 版本」的证明被削弱。 +- CI 成员选择规则为「描述符变更 → 选中所有 mcpp.toml 引用 `sqlite3` 的成员」,故新增第二成员后,改描述符 + 时两个版本都会被 CI 覆盖。 +- SQLite API 纯增量、文件格式兼容,同一行为测试跨版本通常直接通过;各成员的意义在于证明该版本 + 编译+链接+行为均正常。 + +### 8.4 何时加 + +- 用户侧出现对更新版本线的需求(如新 feature / 新文件格式扩展)时按上述步骤追加; +- 加 CN 镜像(获得 `mcpp-res` 写权限)与加新版本相互独立,互不阻塞。 diff --git a/README.md b/README.md index d4bb706..d639f18 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Two kinds of packages live here: | Shape | Examples | |------|------| | Native module library (Form A) | [`mcpplibs.xpkg`](pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](pkgs/f/ffmpeg.lua) (module layer; sources compiled directly through `compat.ffmpeg`) · [`opencv`](pkgs/o/opencv.lua) (single repository: the module layer and the full OpenCV 5 source build both live in the package, and only this descriptor stays on the index side) · [`mcpplibs.grpc`](pkgs/g/grpc.lua) (gRPC 1.83.0 — the one library here that CANNOT be a compat descriptor: upstream publishes no self-contained source artifact, its tag archive carrying abseil/protobuf/re2/boringssl/zlib as empty submodule placeholders, so [grpc-m](https://github.com/mcpplibs/grpc-m)'s release tarball IS that artifact. It vendors only gRPC's own source and takes the five dependencies from this index, so a consumer that also uses protobuf links one copy rather than two) | -| C-source compat (with `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) · [`compat.hiredis`](pkgs/c/compat.hiredis.lua) (the classic 1.2.0 — a 7-TU C build whose flat tarball headers get `hiredis/`-prefixed wrapper headers via `generated_files`, so consumers write `#include ` exactly like upstream's install layout) | +| C-source compat (with `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) · [`compat.hiredis`](pkgs/c/compat.hiredis.lua) (the classic 1.2.0 — a 7-TU C build whose flat tarball headers get `hiredis/`-prefixed wrapper headers via `generated_files`, so consumers write `#include ` exactly like upstream's install layout) · [`compat.sqlite3`](pkgs/c/compat.sqlite3.lua) (plain C-source, no features: the single `sqlite3.c` amalgamation; 3.45.3, the final maintenance release of the most widely deployed 3.45.x line) | | C++-source compat, one depending on the other | [`compat.abseil`](pkgs/c/compat.abseil.lua) (151 TUs; a wildcard over `absl/**` trimmed by upstream's test/benchmark naming conventions) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua) (the libprotobuf runtime, 79 TUs transcribed from upstream's own `src/file_lists.cmake`; declares `compat.abseil` as a dependency because protobuf's public headers include `absl/…`, and its `gzip` feature defines `HAVE_ZLIB` and pulls `compat.zlib`, while `upb` adds protobuf's 64-TU C runtime out of the same tarball. It also exposes **`protoc`** as a `kind = "bin"` target, so a consumer writing `tools = ["protoc"]` gets the compiler built for its own machine out of the same package it links — making a generator/runtime version mismatch inexpressible) · [`compat.re2`](pkgs/c/compat.re2.lua) (22 TUs, upstream's own `RE2_SOURCES`) · [`compat.redis-plus-plus`](pkgs/c/compat.redis-plus-plus.lua) (redis++ 1.3.13 — the sync client, 17 TUs + `patterns/redlock.cpp`, depends on `compat.hiredis`; the one header CMake would generate, `hiredis_features.h`, is snapshotted via `generated_files`, and the async/TLS TUs are left out so the base build stays a two-package pair. Two versions, one on each side of the source-structure watershed, share this ONE source list: 1.3.13 (modern 17-TU layout) and 1.3.3 (pre-`redis_uri.cpp`/`redlock` 15-TU layout) — the union works because 1.3.3's TUs are a strict subset, so exactly two globs match nothing there (a warning, not an error; same trick as compat.catch2)) | | C++-source compat, zero-dep client + optional components | [`compat.websocket`](pkgs/c/compat.websocket.lua) (IXWebSocket 12.0.1 — a pure RFC 6455 client compiled from upstream's `IXWEBSOCKET_SOURCES` minus the four server TUs, so the **base build has zero external dependencies**: TLS off (the OpenSSL/MbedTLS/AppleSSL TUs aren't built) and `IXWEBSOCKET_USE_ZLIB` unset, so the gzip codec compiles to a no-op. Two optional features add on top: `server` (the four server TUs — `IXWebSocketServer`, `IXSocketServer`, `IXHttpServer`, `IXWebSocketProxyServer` — needing nothing external, and it **implies `zlib`** because upstream's server advertises permessage-deflate by default, which the transport negotiates regardless of the define) and `zlib` (deps `compat.zlib` and turns the codec into real per-message-deflate compression). The default-feature test brings its own minimal RFC 6455 echo server on loopback sockets (handshake, masking, fragmentation and close all exercised offline); a second member, `websocket-features`, runs a real `ix::WebSocketServer` and asserts the compression is observable on the wire — a 64 KiB repeated payload round-trips with `wireSize` = 80) | | header-only (with `features`) | [`compat.eigen`](pkgs/c/compat.eigen.lua) | diff --git a/README.zh-CN.md b/README.zh-CN.md index 2ac39c0..b329f3a 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -35,7 +35,7 @@ mcpp self config --mirror CN # 切换至国内镜像,默认使用 GLOBAL 上 | 形态 | 示例 | |------|------| | 原生模块库(Form A) | [`mcpplibs.xpkg`](pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](pkgs/f/ffmpeg.lua)(模块层,源码经 `compat.ffmpeg` 直编) · [`opencv`](pkgs/o/opencv.lua)(单仓库:模块层与 OpenCV 5 全源码构建同在包内,索引侧只留本描述符) · [`mcpplibs.grpc`](pkgs/g/grpc.lua)(gRPC 1.83.0 —— 本索引里唯一**无法**做成 compat 描述符的库:上游不发布任何自包含源码产物,其 tag 归档里 abseil/protobuf/re2/boringssl/zlib 全是空 submodule 占位,因此 [grpc-m](https://github.com/mcpplibs/grpc-m) 的 release tarball 才是那个产物。它只 vendor gRPC 自己的源码,五个依赖全取自本索引,故同时直接使用 protobuf 的消费者链进去的是同一份而非两份)| -| C 源码 compat(含 `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) · [`compat.hiredis`](pkgs/c/compat.hiredis.lua)(经典 1.2.0 —— 7 个 C TU;tarball 平铺头经 `generated_files` 补 `hiredis/` 前缀薄包装头,消费者可写 `#include `,与上游安装布局一致) | +| C 源码 compat(含 `features`) | [`compat.cjson`](pkgs/c/compat.cjson.lua) · [`compat.zlib`](pkgs/c/compat.zlib.lua) · [`compat.hiredis`](pkgs/c/compat.hiredis.lua)(经典 1.2.0 —— 7 个 C TU;tarball 平铺头经 `generated_files` 补 `hiredis/` 前缀薄包装头,消费者可写 `#include `,与上游安装布局一致) · [`compat.sqlite3`](pkgs/c/compat.sqlite3.lua)(纯 C 源码、无 feature:单一 `sqlite3.c` amalgamation;3.45.3,部署最广的 3.45.x 线最后一个维护版) | | C++ 源码 compat(彼此依赖) | [`compat.abseil`](pkgs/c/compat.abseil.lua)(151 TU;对 `absl/**` 取通配后,按上游自身的 test/benchmark 命名约定裁剪) · [`compat.protobuf`](pkgs/c/compat.protobuf.lua)(libprotobuf 运行时,79 TU 逐条转录自上游 `src/file_lists.cmake`;因 protobuf 公开头文件 include 了 `absl/…`,故显式依赖 `compat.abseil`;`gzip` feature 定义 `HAVE_ZLIB` 并拉入 `compat.zlib`,`upb` feature 则从同一个 tarball 里再编出 protobuf 的 64 TU C 运行时;还以 `kind = "bin"` target 暴露 **`protoc`**,消费者写 `tools = ["protoc"]` 即可从「自己链接的那个包」拿到为本机构建的编译器,使生成器与运行时的版本错配无法表达) · [`compat.re2`](pkgs/c/compat.re2.lua)(22 TU,取自上游自身的 `RE2_SOURCES`) · [`compat.redis-plus-plus`](pkgs/c/compat.redis-plus-plus.lua)(redis++ 1.3.13 —— 同步客户端,17 TU + `patterns/redlock.cpp`,依赖 `compat.hiredis`;CMake 唯一会生成的头 `hiredis_features.h` 用 `generated_files` 快照,async/TLS TU 不收,基座保持两包成对。两个版本分处源码结构分水岭两侧,共享同一份源列表:1.3.13(现代 17-TU 布局)与 1.3.3(缺 `redis_uri.cpp`/`redlock` 的 15-TU 旧布局)—— 并集之所以成立,是因为 1.3.3 的 TU 是 1.3.13 的严格子集,恰好两个 glob 在 1.3.3 上零命中(仅警告,非错误;与 compat.catch2 同款手法)) | | C++ 源码 compat(零依赖客户端 + 可选组件) | [`compat.websocket`](pkgs/c/compat.websocket.lua)(IXWebSocket 12.0.1 —— 从上游 `IXWEBSOCKET_SOURCES` 剔掉 4 个 server TU 后直编的纯 RFC 6455 客户端,**基座零外部依赖**:TLS 关闭(OpenSSL/MbedTLS/AppleSSL 三组 TU 均不编),`IXWEBSOCKET_USE_ZLIB` 不定义(gzip codec 编译为 no-op)。两个可选 feature 在基座上叠加:`server`(4 个 server TU —— `IXWebSocketServer`/`IXSocketServer`/`IXHttpServer`/`IXWebSocketProxyServer`,零新增外部依赖,且 **implies `zlib`** —— 因为上游 server 默认就宣称 permessage-deflate,而 transport 的协商不受宏门控)与 `zlib`(依赖 `compat.zlib`,把 codec 变成真正的 permessage-deflate 压缩)。默认构建的测试自带基于 loopback 原始 socket 的最小 RFC 6455 echo server(握手/掩码/分片/关闭全部离线实测);第二个成员 `websocket-features` 跑真实的 `ix::WebSocketServer`,并断言压缩在线路上可观测 —— 64 KiB 重复载荷往返,`wireSize` = 80) | | header-only(含 `features`) | [`compat.eigen`](pkgs/c/compat.eigen.lua) | diff --git a/mcpp.toml b/mcpp.toml index 054476d..945fffc 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -69,6 +69,7 @@ members = [ "tests/examples/llmapi", "tests/examples/md4c", "tests/examples/spdlog-compiled", + "tests/examples/sqlite3", "tests/examples/tinyhttps", "tests/examples/vulkan", "tests/examples/websocket", diff --git a/pkgs/c/compat.sqlite3.lua b/pkgs/c/compat.sqlite3.lua new file mode 100644 index 0000000..a61a201 --- /dev/null +++ b/pkgs/c/compat.sqlite3.lua @@ -0,0 +1,60 @@ +-- Form B inline descriptor for SQLite — the most widely deployed database +-- engine in the world, distributed as a single C amalgamation. Pure-C source +-- build (same shape as compat.cjson / compat.zlib): compile sqlite3.c into a +-- lib, expose sqlite3.h / sqlite3ext.h via include_dirs. shell.c (the +-- interactive CLI) stays out — it pulls in editline/readline dependencies. +-- The amalgamation builds out of the box with no defines. +-- +-- VERSION. 3.45.3 is the FINAL maintenance release of the 3.45.x series, the +-- most widely deployed SQLite line: Ubuntu 24.04 LTS ships 3.45.1 and CPython +-- 3.11/3.12/3.13 official installers ship 3.45.1/3.45.3/3.45.3. Pinning this +-- mature baseline rather than the newest release keeps consumers +-- API-compatible with the largest installed base; newer series can be added +-- later as extra xpm rows (the mcpp block never changes). +-- +-- No CN mirror: sqlite.org is the authoritative source for the amalgamation +-- (the GitHub sqlite/sqlite mirror does NOT carry the generated sqlite3.c), +-- and plain-string url is the documented fallback without mcpp-res write +-- access (docs/cn-mirror.md; precedent: compat.hiredis / compat.spdlog). +package = { + spec = "1", + namespace = "compat", + name = "sqlite3", + description = "SQLite — self-contained SQL database engine (C amalgamation)", + licenses = {"LicenseRef-Public-Domain"}, + repo = "https://sqlite.org/", + type = "package", + + xpm = { + linux = { + ["3.45.3"] = { + url = "https://sqlite.org/2024/sqlite-amalgamation-3450300.zip", + sha256 = "ea170e73e447703e8359308ca2e4366a3ae0c4304a8665896f068c736781c651", + }, + }, + macosx = { + ["3.45.3"] = { + url = "https://sqlite.org/2024/sqlite-amalgamation-3450300.zip", + sha256 = "ea170e73e447703e8359308ca2e4366a3ae0c4304a8665896f068c736781c651", + }, + }, + windows = { + ["3.45.3"] = { + url = "https://sqlite.org/2024/sqlite-amalgamation-3450300.zip", + sha256 = "ea170e73e447703e8359308ca2e4366a3ae0c4304a8665896f068c736781c651", + }, + }, + }, + + mcpp = { + language = "c++23", + import_std = false, + c_standard = "c11", + -- Tarball root: exposes sqlite3.h and sqlite3ext.h to consumers + -- writing `#include `. + include_dirs = { "*" }, + sources = { "*/sqlite3.c" }, + targets = { ["sqlite3"] = { kind = "lib" } }, + deps = { }, + }, +} diff --git a/tests/examples/sqlite3/mcpp.toml b/tests/examples/sqlite3/mcpp.toml new file mode 100644 index 0000000..bc46866 --- /dev/null +++ b/tests/examples/sqlite3/mcpp.toml @@ -0,0 +1,9 @@ +# SQLite test project: depends on compat.sqlite3 (built from source via this +# repo's own index) and asserts behavior under `mcpp test`. Part of the +# mcpp-index self-referential workspace. +[package] +name = "sqlite3-tests" +version = "0.1.0" + +[dependencies.compat] +sqlite3 = "3.45.3" diff --git a/tests/examples/sqlite3/tests/sqlite3_test.cpp b/tests/examples/sqlite3/tests/sqlite3_test.cpp new file mode 100644 index 0000000..fe4df86 --- /dev/null +++ b/tests/examples/sqlite3/tests/sqlite3_test.cpp @@ -0,0 +1,39 @@ +// Behavioral test for compat.sqlite3 (SQLite 3.45.3, C API): assert the +// resolved library version, then open an in-memory database, create a table, +// insert a row, and read it back through both sqlite3_exec and a prepared +// statement. +#include +#include +#include + +int main() { + // The index pins 3.45.3; prove the resolved package is that version. + assert(std::strcmp(sqlite3_libversion(), "3.45.3") == 0); + + sqlite3* db = nullptr; + assert(sqlite3_open(":memory:", &db) == SQLITE_OK); + assert(db != nullptr); + + // DDL + DML through the convenience API. + char* errmsg = nullptr; + int rc = sqlite3_exec(db, + "CREATE TABLE t(id INTEGER PRIMARY KEY, name TEXT NOT NULL);" + "INSERT INTO t(name) VALUES ('mcpp');", + nullptr, nullptr, &errmsg); + assert(rc == SQLITE_OK); + if (errmsg != nullptr) { sqlite3_free(errmsg); errmsg = nullptr; } + + // Read back through a prepared statement. + sqlite3_stmt* stmt = nullptr; + rc = sqlite3_prepare_v2(db, "SELECT id, name FROM t;", -1, &stmt, nullptr); + assert(rc == SQLITE_OK); + assert(stmt != nullptr); + assert(sqlite3_step(stmt) == SQLITE_ROW); + assert(sqlite3_column_int(stmt, 0) == 1); + assert(std::strcmp(reinterpret_cast(sqlite3_column_text(stmt, 1)), "mcpp") == 0); + assert(sqlite3_step(stmt) == SQLITE_DONE); + sqlite3_finalize(stmt); + + assert(sqlite3_close(db) == SQLITE_OK); + return 0; +} From 0de99b16d5c86c14928223671d99410cbfea46b8 Mon Sep 17 00:00:00 2001 From: SPeak Date: Sun, 9 Aug 2026 09:25:19 +0800 Subject: [PATCH 08/12] feat(xpkg): publish libxpkg 0.0.55 (#193) Publish the byte-identical GLOBAL/CN libxpkg 0.0.55 archive on Linux, macOS, and Windows. Required checks passed on 3648230d635e71e55d854d1f9b41446759223f9d. --- pkgs/x/xpkg.lua | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkgs/x/xpkg.lua b/pkgs/x/xpkg.lua index 3d312a2..47931ce 100644 --- a/pkgs/x/xpkg.lua +++ b/pkgs/x/xpkg.lua @@ -12,6 +12,13 @@ package = { xpm = { linux = { + ["0.0.55"] = { + url = { + GLOBAL = "https://github.com/openxlings/libxpkg/archive/refs/tags/0.0.55.tar.gz", + CN = "https://gitcode.com/mcpp-res/xpkg/releases/download/0.0.55/xpkg-0.0.55.tar.gz", + }, + sha256 = "d388c1dd3a531a8e03d304ee2f8dac7a693904daa45de4c400eea7169418fb45", + }, ["0.0.54"] = { url = { GLOBAL = "https://github.com/openxlings/libxpkg/archive/refs/tags/0.0.54.tar.gz", @@ -119,6 +126,13 @@ package = { }, }, macosx = { + ["0.0.55"] = { + url = { + GLOBAL = "https://github.com/openxlings/libxpkg/archive/refs/tags/0.0.55.tar.gz", + CN = "https://gitcode.com/mcpp-res/xpkg/releases/download/0.0.55/xpkg-0.0.55.tar.gz", + }, + sha256 = "d388c1dd3a531a8e03d304ee2f8dac7a693904daa45de4c400eea7169418fb45", + }, ["0.0.54"] = { url = { GLOBAL = "https://github.com/openxlings/libxpkg/archive/refs/tags/0.0.54.tar.gz", @@ -226,6 +240,13 @@ package = { }, }, windows = { + ["0.0.55"] = { + url = { + GLOBAL = "https://github.com/openxlings/libxpkg/archive/refs/tags/0.0.55.tar.gz", + CN = "https://gitcode.com/mcpp-res/xpkg/releases/download/0.0.55/xpkg-0.0.55.tar.gz", + }, + sha256 = "d388c1dd3a531a8e03d304ee2f8dac7a693904daa45de4c400eea7169418fb45", + }, ["0.0.54"] = { url = { GLOBAL = "https://github.com/openxlings/libxpkg/archive/refs/tags/0.0.54.tar.gz", From 401461c21af7180e82dceb5072a0338425226f5c Mon Sep 17 00:00:00 2001 From: wellwei <96378453+wellwei@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:52:53 +0800 Subject: [PATCH 09/12] fix: add immutable libmysqlclient revision (#192) --- pkgs/c/compat.libmysqlclient.lua | 8 ++++++++ pkgs/c/compat.mysql-connector-cpp.lua | 10 +++++----- tests/examples/libmysqlclient/mcpp.toml | 4 ++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/pkgs/c/compat.libmysqlclient.lua b/pkgs/c/compat.libmysqlclient.lua index 22411ea..17de3a7 100644 --- a/pkgs/c/compat.libmysqlclient.lua +++ b/pkgs/c/compat.libmysqlclient.lua @@ -14,12 +14,20 @@ package = { url = "https://github.com/wellwei/libmysqlclient/archive/refs/tags/8.4.6.tar.gz", sha256 = "27decd6716591d84a6f765a3d63fcd24031373841c684369445b83a840b38bea", }, + ["8.4.6.1"] = { + url = "https://github.com/wellwei/libmysqlclient/archive/refs/tags/8.4.6.1.tar.gz", + sha256 = "e50e5da37baa26fff56952d85bd8e33b577be7a534e67df0b8e10a5cc8507af8", + }, }, macosx = { ["8.4.6"] = { url = "https://github.com/wellwei/libmysqlclient/archive/refs/tags/8.4.6.tar.gz", sha256 = "27decd6716591d84a6f765a3d63fcd24031373841c684369445b83a840b38bea", }, + ["8.4.6.1"] = { + url = "https://github.com/wellwei/libmysqlclient/archive/refs/tags/8.4.6.1.tar.gz", + sha256 = "e50e5da37baa26fff56952d85bd8e33b577be7a534e67df0b8e10a5cc8507af8", + }, }, }, diff --git a/pkgs/c/compat.mysql-connector-cpp.lua b/pkgs/c/compat.mysql-connector-cpp.lua index b613bc4..ae6e84c 100644 --- a/pkgs/c/compat.mysql-connector-cpp.lua +++ b/pkgs/c/compat.mysql-connector-cpp.lua @@ -11,7 +11,7 @@ package = { xpm = { linux = { deps = { - "compat:libmysqlclient@8.4.6", + "compat:libmysqlclient@8.4.6.1", "compat:openssl@3.5.1", "xim:cmake@latest", "xim:make@latest", @@ -23,7 +23,7 @@ package = { }, macosx = { deps = { - "compat:libmysqlclient@8.4.6", + "compat:libmysqlclient@8.4.6.1", "compat:openssl@3.5.1", "xim:cmake@latest", }, @@ -41,7 +41,7 @@ package = { include_dirs = { "include" }, targets = { ["mysql_connector_cpp"] = { kind = "lib" } }, deps = { - ["compat.libmysqlclient"] = "8.4.6", + ["compat.libmysqlclient"] = "8.4.6.1", ["compat.openssl"] = "3.5.1", }, @@ -141,7 +141,7 @@ local function find_source_root() end local function find_mysql_source_root(prefix) - local direct = path.join(prefix, "libmysqlclient-8.4.6") + local direct = path.join(prefix, "libmysqlclient-8.4.6.1") if os.isfile(path.join(direct, "mcpp.toml")) then return direct end for _, manifest in ipairs(os.files(path.join(prefix, "**", "mcpp.toml")) or {}) do local root = path.directory(manifest) @@ -198,7 +198,7 @@ function install() .. " deps=" .. tostring(pkginfo.deps_list())) local ok, result = pcall(function() local srcroot = find_source_root() - local mysql = find_dep_install_dir("compat:libmysqlclient", "8.4.6") + local mysql = find_dep_install_dir("compat:libmysqlclient", "8.4.6.1") local openssl = find_dep_install_dir("compat:openssl", "3.5.1") hook_log("srcroot=" .. tostring(srcroot) .. " mysql=" .. tostring(mysql) .. " openssl=" .. tostring(openssl)) diff --git a/tests/examples/libmysqlclient/mcpp.toml b/tests/examples/libmysqlclient/mcpp.toml index f473d3e..4128c8c 100644 --- a/tests/examples/libmysqlclient/mcpp.toml +++ b/tests/examples/libmysqlclient/mcpp.toml @@ -4,13 +4,13 @@ version = "0.1.0" # libmysqlclient 的 mcpp 源码构建目前只验证 Linux/macOS。 [target.'cfg(linux)'.dependencies.compat] -libmysqlclient = "8.4.6" +libmysqlclient = "8.4.6.1" [target.'cfg(linux)'.build] cxxflags = ["-DHAVE_LIBMYSQLCLIENT=1"] [target.'cfg(macos)'.dependencies.compat] -libmysqlclient = "8.4.6" +libmysqlclient = "8.4.6.1" [target.'cfg(macos)'.build] cxxflags = ["-DHAVE_LIBMYSQLCLIENT=1"] From 2dd94f22b8ad820a8279345870b19ce626b55850 Mon Sep 17 00:00:00 2001 From: wellwei <96378453+wellwei@users.noreply.github.com> Date: Sun, 9 Aug 2026 02:54:30 +0800 Subject: [PATCH 10/12] =?UTF-8?q?feat(pkg):=20=E6=96=B0=E5=A2=9E=20compat.?= =?UTF-8?q?gmp=20=E2=80=94=E2=80=94=20GNU=20GMP=206.3.0=20=E9=9D=99?= =?UTF-8?q?=E6=80=81=E5=BA=93(install()=20=E9=A9=B1=E5=8A=A8,linux/macOS)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pkgs/c/compat.gmp.lua:xpkg install() 钩子跑上游 configure && make && make install, 产出 libgmp.a + include/gmp.h(--disable-assembly 通用 C 内核,--libdir 绝对路径)。 - tests/examples/gmp/:mpz/mpq/mpf 三层算术断言,cfg(linux)/cfg(macos) 门控,其他平台 no-op。 - windows 推迟:GMP 无 MSVC 构建路径;镜像待 mcpp-res 权限后补,当前纯字符串上游 url。 - 本地验证:mcpp 2026.8.8.2 macOS arm64 冷启动全流程通过;lint 8 项全过。 - 关联 issue #171。 --- .agents/docs/2026-08-09-add-gmp-plan.md | 89 ++++++++ README.md | 2 +- README.zh-CN.md | 2 +- mcpp.toml | 1 + pkgs/c/compat.gmp.lua | 260 ++++++++++++++++++++++++ tests/examples/gmp/mcpp.toml | 26 +++ tests/examples/gmp/tests/gmp.cpp | 90 ++++++++ 7 files changed, 468 insertions(+), 2 deletions(-) create mode 100644 .agents/docs/2026-08-09-add-gmp-plan.md create mode 100644 pkgs/c/compat.gmp.lua create mode 100644 tests/examples/gmp/mcpp.toml create mode 100644 tests/examples/gmp/tests/gmp.cpp diff --git a/.agents/docs/2026-08-09-add-gmp-plan.md b/.agents/docs/2026-08-09-add-gmp-plan.md new file mode 100644 index 0000000..f92d927 --- /dev/null +++ b/.agents/docs/2026-08-09-add-gmp-plan.md @@ -0,0 +1,89 @@ +# 新增 GMP 收录(compat,install() 驱动构建)+ linux/macOS 方案 + +**日期**: 2026-08-09 +**本仓**: `mcpplibs/mcpp-index` +**issue**: [#171 增加libgmp科学库](https://github.com/mcpplibs/mcpp-index/issues/171)(open,无 assignee/评论) +**目标**: 收录 GNU GMP 6.3.0 为 `compat.gmp`,用户 `#include ` 开箱即用,链接静态 `libgmp.a`。 + +--- + +## 1. 上游调研与形态判定 + +- 最新发布:**6.3.0**(2023-07-30;2026 年仍为最新,ftp.gnu.org 仅有 6.3.0)。GNU 官方 tarball 自带 + `configure`,**不可用 GitHub snapshot**(缺 configure)。 + GLOBAL = `https://ftp.gnu.org/gnu/gmp/gmp-6.3.0.tar.gz`,sha256(两次计算一致) + `e56fd59d76810932a0555aa15a14b61c16bed66110d3c75cc2ac49ddaa9ab24c`。 +- 许可证:GMP 双许可 **LGPL-3.0-or-later OR GPL-2.0-or-later**(tarball README 声明),描述符取宽松侧 + `LGPL-3.0-or-later`。 +- 形态:**autotools 工程**(configure 在构建期生成 `gmp.h`/`config.h`/`gmp-mparam.h`/汇编选择),不属于 + 「列出 .c 文件」的直编形态 → 采用与 `compat.openblas`(Make)、`compat.openssl`(Perl Configure + Make) + 相同的 **xpkg `install()` 钩子**模式:钩子跑上游 `configure && make && make install`,把 `libgmp.a` + + `include/gmp.h` 铺进 install 目录,再 emit anchor TU 触发 mcpp 构建。 + +## 2. 描述符设计(pkgs/c/compat.gmp.lua) + +- 身份:`namespace = "compat"`, `name = "gmp"` → `pkgs/c/`(完整包名首字母)。 +- `mcpp` 段:anchor 源 `mcpp_gmp_anchor.c`(由 install() 生成,非 generated_files),`targets.gmp = lib`, + `include_dirs = {"include"}`,ldflags:linux `-Llib -l:libgmp.a -lm`(libgmp 的 mpf 层用 libm;glibc 不带)、 + macosx `-Llib -lgmp`(ld64 无 `-l:`;libSystem 已含 libm)。 +- configure 参数:`--disable-shared --enable-static --disable-assembly` + `--prefix=` + `--libdir=/lib`。 + - **`--disable-assembly`**:通用 C 内核。与 openblas `TARGET=GENERIC` 同一取舍 —— 可移植、可复现优先, + 牺牲手写汇编加速(后续可逐平台重开;mcpp feature 表无法携带 configure 标志,故这是构建期决策)。 + - **`--libdir` 必须绝对路径**:GMP configure 拒绝相对值(`expected an absolute directory name`), + 与 OpenSSL 不同 —— 首版踩坑,已修复并注释。 +- 平台:linux + macosx(install() 内用 host cc 构建);**windows 推迟**(见 §4)。 +- 构建期依赖:linux `xim:make@latest` + `xim:glibc@>=2.39` + `xim:linux-headers@5.11.1`(cc_override 复用 + compat.openssl 的 libc payload 方案:裸 `cc` 经 xim shim 会注入指向空 subos 的 `--sysroot`,必须显式给 + payload gcc 传 `-isystem/-B/-L`);macosx 无构建期依赖(make 回退 PATH —— macOS 自带 GNU Make 3.81 满足 + GMP 的 >=3.80 要求;cc 钉 `/usr/bin/cc` 以自带 SDK)。 + +## 3. 镜像决策 + +当前无 `mcpp-res` 写权限(gtc 登录为维护者 Sunrisepeak,不擅自发布外部资源),故按 docs/zh/cn-mirror.md +回退:**纯字符串 url**(lint 合规,CN 用户回退上游 ftp.gnu.org)。维护者后续可改写为 +`{ GLOBAL=…, CN=… }`(sha256 不变)。 + +## 4. Windows 推迟 + +GMP **无官方 MSVC 构建路径,也无官方预编译二进制**(vcpkg 在 Windows 用 MPIR fork/MinGW)。与 +`compat.openssl` 现状一致:不声明 windows xpm 块,测试成员用 `cfg(linux)`/`cfg(macos)` 门控 + 其他平台 +no-op main。 + +## 5. feature 评估 + +无可门控组件:GMP 的可选项(汇编、C++ wrapper gmpxx、静态/共享)均由 **configure 标志**控制,而 mcpp +feature 表只能门控 `sources`,不能携带 configure 参数。`--disable-assembly` 作为确定性构建取舍固定写入 +钩子,并在描述符注释说明。 + +## 6. 消费者测试(tests/examples/gmp/) + +- `mcpp.toml`:`[target.'cfg(linux)'/'cfg(macos)'.dependencies.compat] gmp = "6.3.0"` + `-DHAVE_GMP=1`; + windows 无依赖。成员名 `gmp`,已注册进根 `mcpp.toml` members。 +- `tests/gmp.cpp`(TDD 先写断言):覆盖三层算术,参考值用 Python 独立预计算: + - mpz `mpz_fac_ui(100)` = 158 位十进制串(逐字符比对); + - mpz `mpz_powm_ui(2^1000, mod 12345)` = 5341; + - mpz `mpz_gcd(123456789, 987654321)` = 9; + - mpq `mpq_add(1/2, 1/3)` = 5/6; + - mpf `mpf_sqrt(2)` ∈ (1, 2)。 + 每项独立 return code,失败可定位。 + +## 7. 本地验证(mcpp 2026.8.8.2 = validate.yml 当前 pin) + +``` +RED: mcpp test -p gmp → dependency 'compat.gmp': no package found(包缺失,预期失败) +GREEN: mcpp test -p gmp → gmp ... ok / test result ok. 1 passed; 0 failed +冷跑: 清 target/.mcpp 后完整重跑 45.2s(下载→install() configure+make→anchor→链接→运行)→ 通过 +产物: install 目录含 include/gmp.h(84K)、lib/libgmp.a(908K)、mcpp_gmp_anchor.c;链接行 + -L…/xpkgs/compat-x-gmp/6.3.0/lib -lgmp(确认用本包静态库,非宿主 libgmp) +lint: lua5.4 语法/必填字段/无前导 v/check_mirror_urls/check_package_name 全过; + mcpp xpkg parse → compat.gmp,linux+macosx 6.3.0,1 source,1 include,target gmp; + check_cross_package_refs、check_platform_version_parity(全仓)过 +``` + +## 8. 注意事项 / 后续 + +- windows:需上游 MSVC 路径或维护者提供预编译产物,方可补 xpm.windows 块与钩子分支。 +- 汇编:后续可逐平台尝试默认(带 asm)构建,验证 gas 兼容后去掉 `--disable-assembly`;gmpxx(C++ wrapper) + 同理可在 `--enable-cxx` 下评估。 +- CN 镜像:获得 `mcpp-res` 写权限后,用 gtc 建 `mcpp-res/gmp` 并上传同字节 tarball,url 改表形式。 diff --git a/README.md b/README.md index d639f18..558f861 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Two kinds of packages live here: | Host runtime adaptation (drivers are not vendored) | [`compat.glx-runtime`](pkgs/c/compat.glx-runtime.lua) · [`compat.vulkan-runtime`](pkgs/c/compat.vulkan-runtime.lua) (mcpp binaries run against a bundled glibc, so a bare-soname `dlopen` never reaches the host drivers; a symlink farm plus `runtime.library_dirs` bridges that. Note the farm holds only versioned sonames — `library_dirs` also joins the link line) | | Always-on interface define | `CURL_STATICLIB` in [`compat.curl`](pkgs/c/compat.curl.lua): `cflags` is always on but package-private, while a feature's `defines` reaches consumers yet has to be named — `default = { implies = … }` applies unconditionally and happens to give both | | Multiple majors in one package (shape switches with the version) | [`compat.catch2`](pkgs/c/compat.catch2.lua) (3.x compiles `src/catch2/` into a static library; 2.x goes header-only through `single_include/`) | -| External build system (`install()` builds from source) | [`compat.openblas`](pkgs/c/compat.openblas.lua) (Make) · [`compat.openssl`](pkgs/c/compat.openssl.lua) (Perl Configure + Make, static libssl/libcrypto) | +| External build system (`install()` builds from source) | [`compat.openblas`](pkgs/c/compat.openblas.lua) (Make) · [`compat.openssl`](pkgs/c/compat.openssl.lua) (Perl Configure + Make, static libssl/libcrypto) · [`compat.gmp`](pkgs/c/compat.gmp.lua) (GNU configure + Make, static libgmp — portable generic-C kernels, linux/macOS) | | Whole-source direct build (config snapshot + source list, no external build system) | [`compat.ffmpeg`](pkgs/c/compat.ffmpeg.lua) (2281 TUs including NASM assembly, declared through 28 directory globs) | | Module layer over a compat source build (external Form-A repo) | [`godotengine.godot-cpp-m`](pkgs/g/godotengine.godot-cpp-m.lua) (two versions tracking upstream: `10.0.0-rc1` = Godot 4.6, `4.5.0` = Godot 4.5. `import godot_cpp;` re-exports the whole `godot` namespace, ~1800 names GENERATED from the headers rather than curated; the 1022-TU build stays in `compat.godot-cpp`, so the index carries only this descriptor. Macros — `GDCLASS`, `GDREGISTER_CLASS`, `memnew`, `ERR_*` — are the one thing a named module cannot export, so the package ships a side header to include next to the import. It also ships a generated `hashfuncs.hpp` shim — upstream's header minus `static` on two functions whose bodies declare an unnamed union — without which GCC refuses the module interface outright, a hard error no `-W` flag reaches) | | C++23 module wrapper | [`nlohmann.json`](pkgs/n/nlohmann.json.lua) · [`marzer.tomlplusplus`](pkgs/m/marzer.tomlplusplus.lua) · [`neargye.magic_enum`](pkgs/n/neargye.magic_enum.lua) · [`boost-ext.ut`](pkgs/b/boost-ext.ut.lua) (upstream's own `include/boost/ut.cppm` reproduced verbatim but for one `__argc`/`__argv` shim that Clang-on-MSVC needs; namespace `boost-ext` since it is NOT an official Boost library) | diff --git a/README.zh-CN.md b/README.zh-CN.md index b329f3a..c6b27ce 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -49,7 +49,7 @@ mcpp self config --mirror CN # 切换至国内镜像,默认使用 GLOBAL 上 | 宿主运行时适配(不 vendor 驱动) | [`compat.glx-runtime`](pkgs/c/compat.glx-runtime.lua) · [`compat.vulkan-runtime`](pkgs/c/compat.vulkan-runtime.lua)(mcpp 产物跑在自带 glibc 下,裸 soname 的 `dlopen` 够不到宿主驱动;用符号链接农场 + `runtime.library_dirs` 打通。注意 farm 只放带版本号的 soname —— `library_dirs` 同时进链接行) | | 恒开的 interface define | [`compat.curl`](pkgs/c/compat.curl.lua) 的 `CURL_STATICLIB`:`cflags` 恒开但包私有,feature `defines` 可达消费端但需点名 —— `default = { implies = … }` 无条件生效,恰好两者兼得 | | 单包多 major(形态随版本切换) | [`compat.catch2`](pkgs/c/compat.catch2.lua)(3.x 编 `src/catch2/` 出静态库;2.x 走 `single_include/` header-only) | -| 外部构建系统(`install()` 从源码构建) | [`compat.openblas`](pkgs/c/compat.openblas.lua)(Make) · [`compat.openssl`](pkgs/c/compat.openssl.lua)(Perl Configure + Make,静态 libssl/libcrypto) | +| 外部构建系统(`install()` 从源码构建) | [`compat.openblas`](pkgs/c/compat.openblas.lua)(Make) · [`compat.openssl`](pkgs/c/compat.openssl.lua)(Perl Configure + Make,静态 libssl/libcrypto) · [`compat.gmp`](pkgs/c/compat.gmp.lua)(GNU configure + Make,静态 libgmp —— 可移植通用 C 内核,linux/macOS) | | 全源码直编(config 快照 + 源列表,零外部构建系统) | [`compat.ffmpeg`](pkgs/c/compat.ffmpeg.lua)(2281 TU 含 NASM 汇编,28 个目录 glob 声明) | | 模块层叠在 compat 源码构建之上(外部 Form-A 仓) | [`godotengine.godot-cpp-m`](pkgs/g/godotengine.godot-cpp-m.lua)(两个版本与上游对齐:`10.0.0-rc1` 对应 Godot 4.6,`4.5.0` 对应 Godot 4.5。`import godot_cpp;` 重导出整个 `godot` 命名空间,约 1800 个名字由头文件**生成**而非手工罗列;1022 个 TU 的构建留在 `compat.godot-cpp`,索引侧只留这一个描述符。宏 —— `GDCLASS`、`GDREGISTER_CLASS`、`memnew`、`ERR_*` —— 是具名模块唯一带不走的东西,故包内附一个与 import 并排包含的侧头文件。另外还带一份生成的 `hashfuncs.hpp` 遮蔽头 —— 上游那个头去掉两个函数的 `static`(它们体内声明了匿名 union)—— 否则 GCC 直接拒绝该模块接口,且是任何 `-W` 开关都够不到的硬错误) | | C++23 module wrapper | [`nlohmann.json`](pkgs/n/nlohmann.json.lua) · [`marzer.tomlplusplus`](pkgs/m/marzer.tomlplusplus.lua) · [`neargye.magic_enum`](pkgs/n/neargye.magic_enum.lua) · [`boost-ext.ut`](pkgs/b/boost-ext.ut.lua)(逐字复用上游自带的 `include/boost/ut.cppm`,仅加一处 Clang-on-MSVC 需要的 `__argc`/`__argv` shim;命名空间取 `boost-ext`,因其并非 boost 官方库) | diff --git a/mcpp.toml b/mcpp.toml index 945fffc..2d25d10 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -33,6 +33,7 @@ members = [ "tests/examples/ffmpeg", "tests/examples/ffmpeg-module", "tests/examples/fmtlib.fmt", + "tests/examples/gmp", "tests/examples/godot-cpp", "tests/examples/godot-cpp-module", "tests/examples/godot-cpp-module-v10", diff --git a/pkgs/c/compat.gmp.lua b/pkgs/c/compat.gmp.lua new file mode 100644 index 0000000..39dfd0f --- /dev/null +++ b/pkgs/c/compat.gmp.lua @@ -0,0 +1,260 @@ +-- compat.gmp — GNU Multiple Precision Arithmetic Library 6.3.0, built from +-- source as a static C library (libgmp.a + gmp.h). +-- +-- GMP builds through GNU configure + make, which does not fit mcpp's "list the +-- .c files" model: configure generates gmp.h / config.h / gmp-mparam.h from +-- the host probes at build time. So, like compat.openssl and compat.openblas, +-- the xpkg install() hook runs the upstream build and lays the lib + headers +-- under the install dir. mcpp compiles the anchor TU that install() emits, and +-- links the archive via -Llib. +-- +-- Configure flags: +-- * --disable-shared --enable-static — static-only archive. (The consumer +-- example links with -l:libgmp.a on linux, so a stray host libgmp.so.10 +-- cannot be picked up.) +-- * --disable-assembly — portable generic-C kernels. Same tradeoff as +-- compat.openblas's TARGET=GENERIC: correctness/portability first, at the +-- cost of the hand-tuned asm speedups (a later pass could re-enable asm +-- per-platform; mcpp's feature table cannot select configure flags, so +-- this stays a build-time decision). +-- +-- Platforms: linux + macosx. Windows is deferred: GMP has no supported MSVC +-- build path and no official prebuilt binaries (vcpkg uses the MPIR fork or +-- MinGW builds there), so there is no windows xpm entry — the member test +-- carries no dependency and compiles to a no-op main() on windows. +-- +-- Mirror: no CN mirror yet (no authorized mcpp-res write), so the url is the +-- plain upstream string form; maintainers can later rewrite it to +-- { GLOBAL=…, CN=… } with the same sha256. GLOBAL points at ftp.gnu.org — the +-- authoritative release tarball (a GitHub snapshot would lack `configure`). +-- +-- License: dual LGPL-3.0-or-later OR GPL-2.0-or-later (tarball README); the +-- permissive side is declared here. +package = { + spec = "1", + namespace = "compat", + name = "gmp", + description = "GMP — GNU multiple precision arithmetic library (static, install()-driven build)", + licenses = {"LGPL-3.0-or-later"}, + repo = "https://gmplib.org/", + type = "package", + + xpm = { + linux = { + -- glibc + linux-headers are here for the same reason they are on + -- compat.openssl: GMP builds through its own Makefile with a bare + -- `cc`, so every tool it uses has to be something this descriptor + -- resolved, not something it hopes to find. See cc_override(). + deps = { + "xim:make@latest", + "xim:glibc@>=2.39", "xim:linux-headers@5.11.1", + }, + ["6.3.0"] = { + url = "https://ftp.gnu.org/gnu/gmp/gmp-6.3.0.tar.gz", + sha256 = "e56fd59d76810932a0555aa15a14b61c16bed66110d3c75cc2ac49ddaa9ab24c", + }, + }, + macosx = { + -- No xim:make here: that package is linux-only (compat.openblas + -- declares it on macosx and is broken the same way); resolve_make() + -- falls back to PATH (macOS ships GNU Make 3.81, which GMP accepts). + ["6.3.0"] = { + url = "https://ftp.gnu.org/gnu/gmp/gmp-6.3.0.tar.gz", + sha256 = "e56fd59d76810932a0555aa15a14b61c16bed66110d3c75cc2ac49ddaa9ab24c", + }, + }, + -- windows deferred (no MSVC build path upstream) + }, + + mcpp = { + language = "c++23", + import_std = false, + c_standard = "c11", + -- Anchor TU is NOT a generated_files entry: it is emitted by install() + -- so mcpp must run install() (which also builds the lib) before it can + -- compile this source. include/ + lib/ are produced by `make install`. + sources = { "mcpp_gmp_anchor.c" }, + targets = { ["gmp"] = { kind = "lib" } }, + include_dirs = { "include" }, + deps = { }, + + -- libgmp.a pulls in libm (mpf layer uses sqrt/log/exp); libSystem on + -- macOS already carries it, glibc does not. + linux = { ldflags = { "-Llib", "-l:libgmp.a", "-lm" } }, + -- ld64 has no `-l:` spelling; lib/ holds only our archive. + macosx = { ldflags = { "-Llib", "-lgmp" } }, + }, +} + +import("xim.libxpkg.pkginfo") +import("xim.libxpkg.log") + +local function sh_quote(value) + return "'" .. tostring(value):gsub("'", "'\\''") .. "'" +end + +local function resolve_make() + local mk = pkginfo.build_dep("xim:make") or pkginfo.build_dep("make") + if mk and mk.bin then + local cand = path.join(mk.bin, "make") + if os.isfile(cand) then return cand end + end + -- No build dep (macOS): prefer a Homebrew `gmake` when present. macOS's own + -- /usr/bin/make is GNU Make 3.81, which GMP still accepts (needs >= 3.80). + if os.host() == "macosx" and os.isfile("/opt/homebrew/bin/gmake") then + return "/opt/homebrew/bin/gmake" + end + return "make" +end + +-- The C compiler GMP's own build should use. GMP is configured and built +-- outside mcpp's compile rules, so it does not inherit the resolved +-- toolchain's sysroot flags — it just runs `cc`. +-- +-- On macOS the toolchain in PATH is xim's llvm, which has no macOS SDK wired +-- up, so every compile would fail on . Pin Apple's own driver, which +-- finds the SDK by itself. +-- +-- On linux, same story as compat.openssl: PATH reaches the xim gcc through its +-- xvm shim, which injects `--sysroot=` resolved against the CONSUMING +-- project's subos — an empty tree. Hand the payload gcc the three things it +-- needs (headers via -isystem, crt objects via -B, and -lc via -L) from the +-- declared glibc/linux-headers build deps. +local function libc_payloads() + local glibc = pkginfo.build_dep("xim:glibc") or pkginfo.build_dep("glibc") + local kern = pkginfo.build_dep("xim:linux-headers") + or pkginfo.build_dep("linux-headers") + local groot = glibc and glibc.path + local kroot = kern and kern.path + if not (groot and os.isfile(path.join(groot, "include", "stdlib.h"))) then + return nil + end + return groot, kroot +end + +local function cc_override() + if os.host() == "macosx" and os.isfile("/usr/bin/cc") then + return "CC=/usr/bin/cc " + end + if os.host() ~= "linux" then return "" end + + local groot, kroot = libc_payloads() + if not groot then + log.warn("gmp: no xim:glibc payload resolved; leaving CC to PATH" + .. " (fails if the active subos carries no libc headers)") + return "" + end + local libdir = os.isdir(path.join(groot, "lib64")) + and path.join(groot, "lib64") or path.join(groot, "lib") + local cc = "gcc --sysroot=" .. groot + .. " -isystem " .. path.join(groot, "include") + if kroot and os.isdir(path.join(kroot, "include")) then + cc = cc .. " -isystem " .. path.join(kroot, "include") + end + cc = cc .. " -B " .. libdir .. " -L " .. libdir + return "CC=" .. sh_quote(cc) .. " " +end + +-- Last `n` lines of the build log, or nil if it cannot be read. +local function tail_lines(file, n) + local ok, content = pcall(io.readfile, file) + if not ok or not content then return nil end + local lines = {} + for line in (tostring(content) .. "\n"):gmatch("(.-)\n") do + lines[#lines + 1] = line + end + if #lines == 0 then return nil end + return table.concat(lines, "\n", math.max(1, #lines - n + 1), #lines) +end + +-- Run one build step, and on failure print the tail of the log with it. +-- Everything the build says goes to an on-disk log (xim's interface mode +-- swallows subprocess stdout), and xlings surfaces a failed install() as a +-- bare `E_INTERNAL` — so without this the only signal a CI run gives is that +-- something, somewhere, went wrong. +local function run(step, logf, cmd) + local ok, err = pcall(os.exec, string.format("bash -c %s", sh_quote(cmd))) + if ok then return true end + local tail = tail_lines(logf, 40) or "" + log.error("%s", "compat.gmp: " .. step .. " failed (" .. tostring(err) + .. ")\n--- last 40 lines of " .. tostring(logf) .. " ---\n" .. tail) + return false +end + +local function _install_impl() + -- The fetched tarball unpacks to gmp-6.3.0/ beside the archive. + local ifile = pkginfo.install_file() + local srcroot = ifile and tostring(ifile):replace(".tar.gz", "") + or ("gmp-" .. pkginfo.version()) + if not os.isdir(srcroot) then + srcroot = "gmp-" .. pkginfo.version() + end + + -- The prefix is emptied HERE, before the build, so the build log can live + -- inside it: a log that a later os.tryrm would delete, or one left in the + -- transient srcroot, is gone exactly when a failed build needs reading. + local prefix = pkginfo.install_dir() + os.tryrm(prefix) + os.mkdir(prefix) + local logf = path.join(prefix, "mcpp_gmp_build.log") + + local make = resolve_make() + local jobs = (os.default_njob and os.default_njob()) or 4 + -- --disable-assembly: generic C kernels (portable, no m4/gas needed). + -- --libdir=/lib: unlike OpenSSL, GMP's configure REJECTS a + -- relative --libdir ("expected an absolute directory name"); it also + -- derives lib64 on some linux ABIs, so pinning an absolute path also + -- keeps the descriptor's -Llib honest. + local flags = "--disable-shared --enable-static --disable-assembly" + .. " --libdir=" .. path.join(prefix, "lib") + + local cc = cc_override() + if not run("./configure", logf, string.format( + "cd %s && %s./configure --prefix=%s %s >> %s 2>&1", + sh_quote(srcroot), cc, sh_quote(prefix), flags, sh_quote(logf))) then + return false + end + if not run("make", logf, string.format( + "cd %s && %s -j%d >> %s 2>&1", + sh_quote(srcroot), make, jobs, sh_quote(logf))) then + return false + end + if not run("make install", logf, string.format( + "cd %s && %s install >> %s 2>&1", + sh_quote(srcroot), make, sh_quote(logf))) then + return false + end + + -- Verify the build produced the expected archive and header. + local libdir = path.join(prefix, "lib") + local gmp_a = path.join(libdir, "libgmp.a") + local gmp_h = path.join(prefix, "include", "gmp.h") + if not os.isfile(gmp_a) or not os.isfile(gmp_h) then + log.error("compat.gmp: build produced no libgmp.a / include/gmp.h " + .. "under %s (see %s)", prefix, logf) + return false + end + + -- Emit the anchor TU mcpp compiles. Its absence after extraction is what + -- makes mcpp run this install() before the build (same trigger as + -- compat.openblas / compat.openssl / compat.xcb). + io.writefile(path.join(prefix, "mcpp_gmp_anchor.c"), + "int mcpp_compat_gmp_anchor(void) { return 0; }\n") + return true +end + +function install() + -- Windows is deferred: there is no windows xpm block, so version + -- resolution already fails before this point. Kept as a named error in + -- case a windows entry is added before this hook learns to build there. + if os.host() == "windows" then + log.error("compat.gmp: windows is not yet supported") + return false + end + local ok, result = pcall(_install_impl) + if not ok then + log.error("compat.gmp install() failed: %s", tostring(result)) + return false + end + return true +end diff --git a/tests/examples/gmp/mcpp.toml b/tests/examples/gmp/mcpp.toml new file mode 100644 index 0000000..466db96 --- /dev/null +++ b/tests/examples/gmp/mcpp.toml @@ -0,0 +1,26 @@ +# compat.gmp test project — validates the from-source GMP build (static +# libgmp.a + gmp.h) end to end: the install() hook runs upstream's +# configure && make, and this member links the resulting archive and +# exercises the mpz / mpq / mpf layers. +# +# It inherits the workspace-root `compat` redirect, so the descriptor in THIS +# checkout is what gets built and linked. +# +# linux + macOS only: there is no windows xpm entry yet (GMP has no MSVC build +# path), so on windows the member carries no dependency and the test compiles +# to a no-op main(). +[package] +name = "gmp-tests" +version = "0.1.0" + +[target.'cfg(linux)'.dependencies.compat] +gmp = "6.3.0" + +[target.'cfg(linux)'.build] +cxxflags = ["-DHAVE_GMP=1"] + +[target.'cfg(macos)'.dependencies.compat] +gmp = "6.3.0" + +[target.'cfg(macos)'.build] +cxxflags = ["-DHAVE_GMP=1"] diff --git a/tests/examples/gmp/tests/gmp.cpp b/tests/examples/gmp/tests/gmp.cpp new file mode 100644 index 0000000..58824d1 --- /dev/null +++ b/tests/examples/gmp/tests/gmp.cpp @@ -0,0 +1,90 @@ +// compat.gmp end-to-end: gmp.h resolves, libgmp.a links, and the three +// arithmetic layers (mpz integers / mpq rationals / mpf floats) all compute. +// +// The values asserted here are precomputed independently of GMP (Python +// int/fractions/math), so a build that silently picked up a host libgmp, or a +// static archive that is somehow broken, fails instead of passing by accident. +// +// HAVE_GMP comes from this project's own cfg-gated cxxflags — the package is +// linux/macOS-only, so elsewhere this file is an empty main(). +#ifdef HAVE_GMP +#include + +#include +#include + +int main() { + // mpz — factorial: 100! is a known 158-digit number. + { + mpz_t f; + mpz_init(f); + mpz_fac_ui(f, 100); + char* s = mpz_get_str(nullptr, 10, f); + const char* expected = + "93326215443944152681699238856266700490715968264381621468592963895217599993229915" + "608941463976156518286253697920827223758251185210916864000000000000000000000000"; + int ok = s != nullptr && std::strcmp(s, expected) == 0; + if (s != nullptr) std::free(s); + mpz_clear(f); + if (!ok) return 2; + } + + // mpz — modular exponentiation: 2^1000 mod 12345 == 5341. + { + mpz_t base, mod, res; + mpz_init_set_ui(base, 2); + mpz_init_set_ui(mod, 12345); + mpz_init(res); + mpz_powm_ui(res, base, 1000, mod); + int ok = mpz_cmp_ui(res, 5341) == 0; + mpz_clear(base); + mpz_clear(mod); + mpz_clear(res); + if (!ok) return 3; + } + + // mpz — gcd: gcd(123456789, 987654321) == 9. + { + mpz_t a, b, g; + mpz_init_set_ui(a, 123456789); + mpz_init_set_ui(b, 987654321); + mpz_init(g); + mpz_gcd(g, a, b); + int ok = mpz_cmp_ui(g, 9) == 0; + mpz_clear(a); + mpz_clear(b); + mpz_clear(g); + if (!ok) return 4; + } + + // mpq — rational arithmetic: 1/2 + 1/3 == 5/6. + { + mpq_t q, r; + mpq_init(q); + mpq_init(r); + mpq_set_ui(q, 1, 2); + mpq_set_ui(r, 1, 3); + mpq_add(q, q, r); + int ok = mpq_cmp_ui(q, 5, 6) == 0; + mpq_clear(q); + mpq_clear(r); + if (!ok) return 5; + } + + // mpf — floating layer: 1 < sqrt(2) < 2 (mpf_sqrt on a double seed). + { + mpf_t x, s; + mpf_init_set_d(x, 2.0); + mpf_init(s); + mpf_sqrt(s, x); + int ok = mpf_cmp_d(s, 1.0) > 0 && mpf_cmp_d(s, 2.0) < 0; + mpf_clear(x); + mpf_clear(s); + if (!ok) return 6; + } + + return 0; +} +#else +int main() { return 0; } // compat.gmp is linux/macOS-only; no-op elsewhere +#endif From 9b28bd8198696b418eb93475362c05cba800bc99 Mon Sep 17 00:00:00 2001 From: wellwei <96378453+wellwei@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:02:08 +0800 Subject: [PATCH 11/12] =?UTF-8?q?fix(pkg):=20compat.gmp=20=E5=9C=A8=20linu?= =?UTF-8?q?x=20=E6=94=B9=E7=94=A8=E5=AE=BF=E4=B8=BB=20gcc=20=E2=80=94?= =?UTF-8?q?=E2=80=94=20GMP=20configure=20=E9=9C=80=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E5=B9=B6=E8=BF=90=E8=A1=8C=E6=8E=A2=E9=92=88/=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit linux default 腿 CI 实测失败:configure 报 could not find a working compiler。 GMP 的 configure 会编译并运行 ABI 探针、make 期运行 gen-* 工具,链接到 xim glibc payload(--sysroot/-B/-L)的这类可执行文件在本钩子环境无法运行;openssl 的 Configure 只写 Makefile 所以无此问题。改 /usr/bin/gcc 后探针/工具跑宿主 glibc, 静态库符号由消费端工具链更新的 payload glibc 在最终链接时满足。 --- .agents/docs/2026-08-09-add-gmp-plan.md | 19 +++++-- pkgs/c/compat.gmp.lua | 69 ++++++++++--------------- 2 files changed, 42 insertions(+), 46 deletions(-) diff --git a/.agents/docs/2026-08-09-add-gmp-plan.md b/.agents/docs/2026-08-09-add-gmp-plan.md index f92d927..eb0ef5a 100644 --- a/.agents/docs/2026-08-09-add-gmp-plan.md +++ b/.agents/docs/2026-08-09-add-gmp-plan.md @@ -33,10 +33,16 @@ - **`--libdir` 必须绝对路径**:GMP configure 拒绝相对值(`expected an absolute directory name`), 与 OpenSSL 不同 —— 首版踩坑,已修复并注释。 - 平台:linux + macosx(install() 内用 host cc 构建);**windows 推迟**(见 §4)。 -- 构建期依赖:linux `xim:make@latest` + `xim:glibc@>=2.39` + `xim:linux-headers@5.11.1`(cc_override 复用 - compat.openssl 的 libc payload 方案:裸 `cc` 经 xim shim 会注入指向空 subos 的 `--sysroot`,必须显式给 - payload gcc 传 `-isystem/-B/-L`);macosx 无构建期依赖(make 回退 PATH —— macOS 自带 GNU Make 3.81 满足 - GMP 的 >=3.80 要求;cc 钉 `/usr/bin/cc` 以自带 SDK)。 +- 构建期依赖:linux `xim:make@latest`(GNU make);macosx 无(make 回退 PATH —— macOS 自带 GNU Make 3.81 满足 + GMP 的 >=3.80 要求)。 +- **编译器用宿主 cc**(linux `/usr/bin/gcc`、macOS `/usr/bin/cc`),不用 xim payload 工具链: + - macOS 与 openssl 一致:xim llvm 无 macOS SDK,钉 Apple 自带驱动。 + - linux 与 openssl **不同**且更严:openssl 的 Configure 只写 Makefile 从不运行可执行文件,所以 payload + glibc 的 `--sysroot/-B/-L` 足够;但 GMP 的 configure 会**编译并运行** ABI 探针程序,`make` 还要运行 + gen-bases/gen-fib/gen-psqr 等构建工具 —— 链接到 xim glibc payload 的这类二进制在本钩子环境里 + **无法运行**,configure 报 `could not find a working compiler`(2026-08-09 linux CI 实测)。 + 宿主 gcc 的探针/工具直接跑宿主 glibc,静态库的 glibc/libm 符号由消费端工具链**更新的** payload glibc + (2.44 > 宿主 2.39)在最终链接时满足 —— 与 openssl macOS 腿(Apple clang 构建、xim llvm 消费)同一模型。 ## 3. 镜像决策 @@ -79,6 +85,11 @@ GREEN: mcpp test -p gmp → gmp ... ok / test result ok. 1 passed; 0 failed lint: lua5.4 语法/必填字段/无前导 v/check_mirror_urls/check_package_name 全过; mcpp xpkg parse → compat.gmp,linux+macosx 6.3.0,1 source,1 include,target gmp; check_cross_package_refs、check_platform_version_parity(全仓)过 + +CI(wellwei/mcpp-index#4,选跑 gmp 成员): +- linux default 首次失败:GMP configure `could not find a working compiler` —— payload glibc 链接的 + 探针/工具不可运行(见 §2 编译器说明)→ 改宿主 gcc 后重跑。 +- 待 linux(gcc+llvm)/macos/windows 全绿后收尾。 ``` ## 8. 注意事项 / 后续 diff --git a/pkgs/c/compat.gmp.lua b/pkgs/c/compat.gmp.lua index 39dfd0f..6d11ae3 100644 --- a/pkgs/c/compat.gmp.lua +++ b/pkgs/c/compat.gmp.lua @@ -23,6 +23,12 @@ -- MinGW builds there), so there is no windows xpm entry — the member test -- carries no dependency and compiles to a no-op main() on windows. -- +-- Compiler: GMP builds with the HOST cc (/usr/bin/gcc on linux, /usr/bin/cc +-- on macOS). GMP's configure runs probe executables and make runs gen-* tools, +-- so they must be runnable on the host; the xim payload toolchain cannot +-- produce those (see cc_override()). The static archive's glibc/libm symbols +-- resolve at final link against the consumer toolchain's newer payload glibc. +-- -- Mirror: no CN mirror yet (no authorized mcpp-res write), so the url is the -- plain upstream string form; maintainers can later rewrite it to -- { GLOBAL=…, CN=… } with the same sha256. GLOBAL points at ftp.gnu.org — the @@ -41,14 +47,10 @@ package = { xpm = { linux = { - -- glibc + linux-headers are here for the same reason they are on - -- compat.openssl: GMP builds through its own Makefile with a bare - -- `cc`, so every tool it uses has to be something this descriptor - -- resolved, not something it hopes to find. See cc_override(). - deps = { - "xim:make@latest", - "xim:glibc@>=2.39", "xim:linux-headers@5.11.1", - }, + -- GNU make comes from the xim:make build dep; the C compiler is the + -- HOST gcc (/usr/bin/gcc) — see cc_override() for why not the xim + -- payload toolchain. + deps = { "xim:make@latest" }, ["6.3.0"] = { url = "https://ftp.gnu.org/gnu/gmp/gmp-6.3.0.tar.gz", sha256 = "e56fd59d76810932a0555aa15a14b61c16bed66110d3c75cc2ac49ddaa9ab24c", @@ -109,50 +111,33 @@ end -- The C compiler GMP's own build should use. GMP is configured and built -- outside mcpp's compile rules, so it does not inherit the resolved --- toolchain's sysroot flags — it just runs `cc`. +-- toolchain's flags — it just runs `cc`. -- -- On macOS the toolchain in PATH is xim's llvm, which has no macOS SDK wired -- up, so every compile would fail on . Pin Apple's own driver, which -- finds the SDK by itself. -- --- On linux, same story as compat.openssl: PATH reaches the xim gcc through its --- xvm shim, which injects `--sysroot=` resolved against the CONSUMING --- project's subos — an empty tree. Hand the payload gcc the three things it --- needs (headers via -isystem, crt objects via -B, and -lc via -L) from the --- declared glibc/linux-headers build deps. -local function libc_payloads() - local glibc = pkginfo.build_dep("xim:glibc") or pkginfo.build_dep("glibc") - local kern = pkginfo.build_dep("xim:linux-headers") - or pkginfo.build_dep("linux-headers") - local groot = glibc and glibc.path - local kroot = kern and kern.path - if not (groot and os.isfile(path.join(groot, "include", "stdlib.h"))) then - return nil - end - return groot, kroot -end - +-- On linux, unlike compat.openssl, the xim payload gcc is NOT usable: +-- OpenSSL's Configure only writes Makefiles, but GMP's configure COMPILES AND +-- RUNS probe programs (ABI detection) and `make` runs gen-* build tools. A +-- conftest/gen-* binary linked against the xim glibc payload +-- (--sysroot/-B/-L) is not runnable in this hook, so configure reports +-- "could not find a working compiler" (seen on CI 2026-08-09). Use the HOST +-- /usr/bin/gcc instead: its probes and build tools run against the host +-- glibc, and the static archive's glibc symbols are satisfied at final link +-- by the consumer toolchain's own (newer) glibc payload — the same model as +-- compat.openssl's macOS leg (Apple clang builds, xim llvm consumes). local function cc_override() if os.host() == "macosx" and os.isfile("/usr/bin/cc") then return "CC=/usr/bin/cc " end - if os.host() ~= "linux" then return "" end - - local groot, kroot = libc_payloads() - if not groot then - log.warn("gmp: no xim:glibc payload resolved; leaving CC to PATH" - .. " (fails if the active subos carries no libc headers)") - return "" + if os.host() == "linux" and os.isfile("/usr/bin/gcc") then + return "CC=/usr/bin/gcc " end - local libdir = os.isdir(path.join(groot, "lib64")) - and path.join(groot, "lib64") or path.join(groot, "lib") - local cc = "gcc --sysroot=" .. groot - .. " -isystem " .. path.join(groot, "include") - if kroot and os.isdir(path.join(kroot, "include")) then - cc = cc .. " -isystem " .. path.join(kroot, "include") - end - cc = cc .. " -B " .. libdir .. " -L " .. libdir - return "CC=" .. sh_quote(cc) .. " " + if os.host() ~= "linux" then return "" end + log.warn("gmp: /usr/bin/gcc not found; leaving CC to PATH" + .. " (likely fails: xim's gcc shim injects an empty --sysroot)") + return "" end -- Last `n` lines of the build log, or nil if it cannot be read. From 385ab10b4ef29a52ffa754d964a783004bae99ca Mon Sep 17 00:00:00 2001 From: wellwei <96378453+wellwei@users.noreply.github.com> Date: Sun, 9 Aug 2026 14:22:28 +0800 Subject: [PATCH 12/12] =?UTF-8?q?docs:=20=E8=AE=B0=E5=BD=95=20compat.gmp?= =?UTF-8?q?=20=E7=9A=84=20fork=20CI=20=E6=9C=80=E7=BB=88=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E7=BB=93=E8=AE=BA(=E5=9B=9B=E6=9D=A1=20workspace=20=E8=85=BF?= =?UTF-8?q?=E5=85=A8=E7=BB=BF;mirror=20=E7=BA=A2=E4=B8=BA=E6=97=A2?= =?UTF-8?q?=E6=9C=89=20opencv=20403)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .agents/docs/2026-08-09-add-gmp-plan.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.agents/docs/2026-08-09-add-gmp-plan.md b/.agents/docs/2026-08-09-add-gmp-plan.md index eb0ef5a..9802ee4 100644 --- a/.agents/docs/2026-08-09-add-gmp-plan.md +++ b/.agents/docs/2026-08-09-add-gmp-plan.md @@ -86,10 +86,13 @@ lint: lua5.4 语法/必填字段/无前导 v/check_mirror_urls/check_package_na mcpp xpkg parse → compat.gmp,linux+macosx 6.3.0,1 source,1 include,target gmp; check_cross_package_refs、check_platform_version_parity(全仓)过 -CI(wellwei/mcpp-index#4,选跑 gmp 成员): -- linux default 首次失败:GMP configure `could not find a working compiler` —— payload glibc 链接的 +CI(wellwei/mcpp-index#4,选跑 gmp 成员)最终结论: +- linux default 首轮失败:GMP configure `could not find a working compiler` —— payload glibc 链接的 探针/工具不可运行(见 §2 编译器说明)→ 改宿主 gcc 后重跑。 -- 待 linux(gcc+llvm)/macos/windows 全绿后收尾。 +- 重跑后**四条 workspace 腿全绿**:linux default ✅(1m19s)、linux llvm ✅(1m29s)、macos ✅(1m41s)、 + windows ✅(1m23s),另有 lint/select/graphics/timings ✅。 +- 唯一红 job 为 `mirror-cn-reachable`:既有 opencv 5.0.0 CN 资产(origin/main 上已存在,本分支未触碰 + pkgs/o/)被 gitcode 返回 403,与 gmp 无关。 ``` ## 8. 注意事项 / 后续