From 5bf1f2d6662f77d753d3b88d9919ac55b8f5686c Mon Sep 17 00:00:00 2001 From: Remi Thebault Date: Thu, 18 May 2023 08:43:00 +0200 Subject: [PATCH 1/8] stabilize the cache directory structure --- source/dub/dub.d | 5 +++++ source/dub/generators/generator.d | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/source/dub/dub.d b/source/dub/dub.d index 28ada243e..87a78bc9f 100644 --- a/source/dub/dub.d +++ b/source/dub/dub.d @@ -1883,6 +1883,11 @@ private struct SpecialDirs { * project directory, but this led to issues with packages stored on * read-only file system / location, and lingering artifacts scattered * through the file system. + * + * Dub writes in the cache directory some Json description files + * of the available artifacts. These files are intended to be read by + * 3rd party software (e.g. Meson). The default cache location specified + * in this function should therefore not change across future Dub versions. */ NativePath cache; diff --git a/source/dub/generators/generator.d b/source/dub/generators/generator.d index 8ff77d0ec..3e0914b70 100644 --- a/source/dub/generators/generator.d +++ b/source/dub/generators/generator.d @@ -771,9 +771,15 @@ class ProjectGenerator /** * Compute and returns the path were artifacts are stored for a given package * - * Artifacts are usually stored in: + * Artifacts are stored in: * `$DUB_HOME/cache/$PKG_NAME/$PKG_VERSION[/+$SUB_PKG_NAME]/` * Note that the leading `+` in the sub-package name is to avoid any ambiguity. + * + * Dub writes in the returned path a Json description file of the available + * artifacts in this cache location. This Json file is read by 3rd party + * software (e.g. Meson). Returned path should therefore not change across + * future Dub versions. + * * Build artifacts are usually stored in a sub-folder named "build", * as their names are based on user-supplied values. * From 9b63683ed41e5604a3543b87be1929195cab8041 Mon Sep 17 00:00:00 2001 From: Remi Thebault Date: Thu, 18 May 2023 08:51:29 +0200 Subject: [PATCH 2/8] write cache database Json file --- source/dub/generators/build.d | 49 ++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/source/dub/generators/build.d b/source/dub/generators/build.d index 8583eab84..a0e5368fb 100644 --- a/source/dub/generators/build.d +++ b/source/dub/generators/build.d @@ -285,12 +285,59 @@ class BuildGenerator : ProjectGenerator { buildWithCompiler(settings, cbuildsettings); target_binary_path = getTargetPath(cbuildsettings, settings); - if (!settings.tempBuild) + if (!settings.tempBuild) { copyTargetFile(target_path, buildsettings, settings); + updateCacheDatabase(settings, cbuildsettings, pack, config, build_id, target_binary_path.toNativeString()); + } return false; } + private void updateCacheDatabase(GeneratorSettings settings, BuildSettings buildsettings, in Package pack, string config, + string build_id, string target_binary_path) + { + import dub.internal.vibecompat.data.json; + import core.time : seconds; + + // Generate a `build-db.json` in the package version cache directory. + // This is read by 3rd party software (e.g. Meson) in order to find + // relevant build artifacts in Dub's cache. + + enum jsonFileName = "build-db.json"; + enum lockFileName = "build-db.lock"; + + const pkgCacheDir = packageCache(settings.cache, pack); + auto lock = lockFile((pkgCacheDir ~ lockFileName).toNativeString(), 3.seconds); + + const dbPath = pkgCacheDir ~ jsonFileName; + const dbPathStr = dbPath.toNativeString(); + Json db; + if (exists(dbPathStr)) { + const text = stripUTF8Bom(cast(string)readFile(dbPath)); + db = parseJsonString(text, dbPathStr); + enforce(db.type == Json.Type.array, "Expected a JSON array in " ~ dbPathStr); + } + else { + db = Json.emptyArray; + } + Json entry = Json.emptyObject; + + entry["package"] = pack.name; + entry["configuration"] = config; + entry["buildType"] = settings.buildType; + entry["compiler"] = settings.platform.compiler; + entry["compilerVersion"] = settings.platform.compilerVersion; + entry["compilerBinary"] = settings.platform.compilerBinary; + entry["architecture"] = serializeToJson(settings.platform.architecture); + entry["platform"] = serializeToJson(settings.platform.platform); + entry["buildId"] = build_id; + entry["targetBinaryPath"] = target_binary_path; + + db ~= entry; + + writeFile(dbPath, representation(db.toPrettyString())); + } + private void performRDMDBuild(GeneratorSettings settings, ref BuildSettings buildsettings, in Package pack, string config, out NativePath target_path) { auto cwd = settings.toolWorkingDirectory; From b6a2b67ab4a807f281ec1bd29de78d9f29d5ec39 Mon Sep 17 00:00:00 2001 From: Remi Thebault Date: Thu, 18 May 2023 11:50:19 +0200 Subject: [PATCH 3/8] avoid duplicates in cache DB --- source/dub/generators/build.d | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/dub/generators/build.d b/source/dub/generators/build.d index a0e5368fb..8c2af1635 100644 --- a/source/dub/generators/build.d +++ b/source/dub/generators/build.d @@ -320,6 +320,14 @@ class BuildGenerator : ProjectGenerator { else { db = Json.emptyArray; } + + foreach (entry; db) { + if (entry["buildId"].get!string == build_id) { + // duplicate + return; + } + } + Json entry = Json.emptyObject; entry["package"] = pack.name; From e752171970dd94ef175d81f10914d0367569c0aa Mon Sep 17 00:00:00 2001 From: Remi Thebault Date: Thu, 18 May 2023 16:04:17 +0200 Subject: [PATCH 4/8] add "version" to the cache database --- source/dub/generators/build.d | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/dub/generators/build.d b/source/dub/generators/build.d index 8c2af1635..8735bb93b 100644 --- a/source/dub/generators/build.d +++ b/source/dub/generators/build.d @@ -330,16 +330,17 @@ class BuildGenerator : ProjectGenerator { Json entry = Json.emptyObject; - entry["package"] = pack.name; - entry["configuration"] = config; + entry["architecture"] = serializeToJson(settings.platform.architecture); + entry["buildId"] = build_id; entry["buildType"] = settings.buildType; entry["compiler"] = settings.platform.compiler; - entry["compilerVersion"] = settings.platform.compilerVersion; entry["compilerBinary"] = settings.platform.compilerBinary; - entry["architecture"] = serializeToJson(settings.platform.architecture); + entry["compilerVersion"] = settings.platform.compilerVersion; + entry["configuration"] = config; + entry["package"] = pack.name; entry["platform"] = serializeToJson(settings.platform.platform); - entry["buildId"] = build_id; entry["targetBinaryPath"] = target_binary_path; + entry["version"] = pack.version_.toString(); db ~= entry; From 06e237f74913f4667359db5f7482a4cad237f8a9 Mon Sep 17 00:00:00 2001 From: Remi Thebault Date: Thu, 18 May 2023 16:06:02 +0200 Subject: [PATCH 5/8] change db filename to "db.json" --- source/dub/generators/build.d | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dub/generators/build.d b/source/dub/generators/build.d index 8735bb93b..62d4510bf 100644 --- a/source/dub/generators/build.d +++ b/source/dub/generators/build.d @@ -299,12 +299,12 @@ class BuildGenerator : ProjectGenerator { import dub.internal.vibecompat.data.json; import core.time : seconds; - // Generate a `build-db.json` in the package version cache directory. + // Generate a `db.json` in the package version cache directory. // This is read by 3rd party software (e.g. Meson) in order to find // relevant build artifacts in Dub's cache. - enum jsonFileName = "build-db.json"; - enum lockFileName = "build-db.lock"; + enum jsonFileName = "db.json"; + enum lockFileName = "db.lock"; const pkgCacheDir = packageCache(settings.cache, pack); auto lock = lockFile((pkgCacheDir ~ lockFileName).toNativeString(), 3.seconds); From 4328c631b3ad273762bff854223cbbd45b86a124 Mon Sep 17 00:00:00 2001 From: Remi Thebault Date: Thu, 18 May 2023 16:14:40 +0200 Subject: [PATCH 6/8] add test for the cache database --- test/pr2642-cache-db/.gitignore | 2 + test/pr2642-cache-db/.no_test | 0 test/pr2642-cache-db/dub.sdl | 2 + test/pr2642-cache-db/source/test_cache_db.d | 78 +++++++++++++++++++++ 4 files changed, 82 insertions(+) create mode 100644 test/pr2642-cache-db/.gitignore create mode 100644 test/pr2642-cache-db/.no_test create mode 100644 test/pr2642-cache-db/dub.sdl create mode 100644 test/pr2642-cache-db/source/test_cache_db.d diff --git a/test/pr2642-cache-db/.gitignore b/test/pr2642-cache-db/.gitignore new file mode 100644 index 000000000..da01cd648 --- /dev/null +++ b/test/pr2642-cache-db/.gitignore @@ -0,0 +1,2 @@ +dubhome/ +pr2642-cache-db diff --git a/test/pr2642-cache-db/.no_test b/test/pr2642-cache-db/.no_test new file mode 100644 index 000000000..e69de29bb diff --git a/test/pr2642-cache-db/dub.sdl b/test/pr2642-cache-db/dub.sdl new file mode 100644 index 000000000..f9fca89bc --- /dev/null +++ b/test/pr2642-cache-db/dub.sdl @@ -0,0 +1,2 @@ +name "pr2642-cache-db"; +targetType "executable"; diff --git a/test/pr2642-cache-db/source/test_cache_db.d b/test/pr2642-cache-db/source/test_cache_db.d new file mode 100644 index 000000000..662954d49 --- /dev/null +++ b/test/pr2642-cache-db/source/test_cache_db.d @@ -0,0 +1,78 @@ +module test_cache_db; + +import std.path; +import std.file; +import std.process; +import std.stdio; +import std.json; + +void main() +{ + const dubhome = __FILE_FULL_PATH__.dirName().dirName().buildNormalizedPath("dubhome"); + if (exists(dubhome)) + { + rmdirRecurse(dubhome); + } + + const string[string] env = [ + "DUB_HOME": dubhome, + ]; + const fetchProgram = [ + environment["DUB"], + "fetch", + "vibe-d@0.9.6", + ]; + auto dubFetch = spawnProcess(fetchProgram, stdin, stdout, stderr, env); + wait(dubFetch); + const buildProgram = [ + environment["DUB"], + "build", + "--build=debug", + "vibe-d:http@0.9.6", + ]; + auto dubBuild = spawnProcess(buildProgram, stdin, stdout, stderr, env); + wait(dubBuild); + + scope (success) + { + // leave dubhome in the tree for analysis in case of failure + rmdirRecurse(dubhome); + } + + const buildDbPath = buildNormalizedPath(dubhome, "cache", "vibe-d", "0.9.6", "+http", "db.json"); + assert(exists(buildDbPath), buildDbPath ~ " should exist"); + const buildDbStr = readText(buildDbPath); + auto json = parseJSON(buildDbStr); + assert(json.type == JSONType.array, "build db should be an array"); + assert(json.array.length == 1); + auto db = json.array[0].object; + + void assertArray(string field) + { + assert(field in db, "db.json should have an array field " ~ field); + assert(db[field].type == JSONType.array, "expected field " ~ field ~ " to be an array"); + } + + void assertString(string field, string value = null) + { + assert(field in db, "db.json should have an string field " ~ field); + assert(db[field].type == JSONType.string, "expected field " ~ field ~ " to be a string"); + if (value) + assert(db[field].str == value, "expected field " ~ field ~ " to equal " ~ value); + } + + assertArray("architecture"); + assertString("buildId"); + assertString("buildType", "debug"); + assertString("compiler"); + assertString("compilerBinary"); + assertString("compilerVersion"); + assertString("configuration", "library"); + assertString("package", "vibe-d:http"); + assertArray("platform"); + assertString("targetBinaryPath"); + assertString("version", "0.9.6"); + + const binName = db["targetBinaryPath"].str; + assert(isFile(binName), "expected " ~ binName ~ " to be a file."); +} From 8323d1f74bf1a1c9acef7a10ba12a303ab6132e9 Mon Sep 17 00:00:00 2001 From: Remi Thebault Date: Thu, 18 May 2023 16:34:01 +0200 Subject: [PATCH 7/8] cache db test uses gitcompatibledubpackage --- test/pr2642-cache-db/source/test_cache_db.d | 50 ++++++++++++++++----- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/test/pr2642-cache-db/source/test_cache_db.d b/test/pr2642-cache-db/source/test_cache_db.d index 662954d49..28a2438ac 100644 --- a/test/pr2642-cache-db/source/test_cache_db.d +++ b/test/pr2642-cache-db/source/test_cache_db.d @@ -20,17 +20,29 @@ void main() const fetchProgram = [ environment["DUB"], "fetch", - "vibe-d@0.9.6", + "gitcompatibledubpackage@1.0.4", ]; auto dubFetch = spawnProcess(fetchProgram, stdin, stdout, stderr, env); wait(dubFetch); - const buildProgram = [ + + const buildProgramLib = [ + environment["DUB"], + "build", + "--build=debug", + "--config=lib", + "gitcompatibledubpackage@1.0.4", + ]; + auto dubBuild = spawnProcess(buildProgramLib, stdin, stdout, stderr, env); + wait(dubBuild); + + const buildProgramExe = [ environment["DUB"], "build", "--build=debug", - "vibe-d:http@0.9.6", + "--config=exe", + "gitcompatibledubpackage@1.0.4", ]; - auto dubBuild = spawnProcess(buildProgram, stdin, stdout, stderr, env); + dubBuild = spawnProcess(buildProgramExe, stdin, stdout, stderr, env); wait(dubBuild); scope (success) @@ -39,12 +51,13 @@ void main() rmdirRecurse(dubhome); } - const buildDbPath = buildNormalizedPath(dubhome, "cache", "vibe-d", "0.9.6", "+http", "db.json"); + const buildDbPath = buildNormalizedPath(dubhome, "cache", "gitcompatibledubpackage", "1.0.4", "db.json"); assert(exists(buildDbPath), buildDbPath ~ " should exist"); const buildDbStr = readText(buildDbPath); auto json = parseJSON(buildDbStr); assert(json.type == JSONType.array, "build db should be an array"); - assert(json.array.length == 1); + assert(json.array.length == 2, "build db should have 2 entries"); + auto db = json.array[0].object; void assertArray(string field) @@ -67,12 +80,29 @@ void main() assertString("compiler"); assertString("compilerBinary"); assertString("compilerVersion"); - assertString("configuration", "library"); - assertString("package", "vibe-d:http"); + assertString("configuration", "lib"); + assertString("package", "gitcompatibledubpackage"); + assertArray("platform"); + assertString("targetBinaryPath"); + assertString("version", "1.0.4"); + + auto binName = db["targetBinaryPath"].str; + assert(isFile(binName), "expected " ~ binName ~ " to be a file."); + + db = json.array[1].object; + + assertArray("architecture"); + assertString("buildId"); + assertString("buildType", "debug"); + assertString("compiler"); + assertString("compilerBinary"); + assertString("compilerVersion"); + assertString("configuration", "exe"); + assertString("package", "gitcompatibledubpackage"); assertArray("platform"); assertString("targetBinaryPath"); - assertString("version", "0.9.6"); + assertString("version", "1.0.4"); - const binName = db["targetBinaryPath"].str; + binName = db["targetBinaryPath"].str; assert(isFile(binName), "expected " ~ binName ~ " to be a file."); } From 18834c0171ef77d7e7df0c53a360d29d0c140aed Mon Sep 17 00:00:00 2001 From: Remi Thebault Date: Thu, 18 May 2023 16:35:48 +0200 Subject: [PATCH 8/8] checking for duplicates in reverse order cache hits are more likely in the most recent builds --- source/dub/generators/build.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dub/generators/build.d b/source/dub/generators/build.d index 62d4510bf..73fe18b52 100644 --- a/source/dub/generators/build.d +++ b/source/dub/generators/build.d @@ -321,7 +321,7 @@ class BuildGenerator : ProjectGenerator { db = Json.emptyArray; } - foreach (entry; db) { + foreach_reverse (entry; db) { if (entry["buildId"].get!string == build_id) { // duplicate return;