[WIP] Hash dependent build

.github/workflows/main.yml

@@ -74,4 +74,5 @@ jobs:
       run: |
         dub build --compiler=${{ env.DC }}
         dub test  --compiler=${{ env.DC }}
+        .\bin\dub --compiler=${{ env.DC }} --single scripts\ci\hash_dependent_build.d
         dub run   --compiler=${{ env.DC }} --single test\issue_2051_running_unittests_from_dub_single_file_packages_fails.d

.gitignore

@@ -30,6 +30,7 @@ __dummy.html
 /test/expected-issue616-output
 /test/describe-project/dummy.dat
 /test/describe-project/dummy-dep1.dat
+/test/hash-dependent-build
 /test/*/main/main
 /test/*/*test-library
 /test/*/*test-application
@@ -41,3 +42,5 @@ cov/
 # Ignore auto-generated docs
 /docs
 scripts/man/dub*.1.gz
+*.exe
+bin/*.exe

changelog/hash-dependent-build.dd

@@ -0,0 +1,3 @@
+Added hash dependent build
+
+dub calculates hash sum of source files to detect if any of source files has been changed to trigger rebuilding.

scripts/ci/hash_dependent_build.d

@@ -0,0 +1,216 @@
+/+ dub.sdl:
+   name "hash"
+ +/
+
+import std.algorithm : any;
+import std.array : array;
+import std.string : lineSplitter;
+import std.datetime : dur, SysTime;
+import std.file;
+import std.format : format;
+import std.path;
+import std.process;
+import std.stdio : stderr, writeln;
+
+enum TestProjectName = "hash-dependent-build";
+immutable source_name = "source/app.d";
+version(Windows) immutable artifact_name = TestProjectName ~ ".exe";
+else             immutable artifact_name = TestProjectName;
+
+enum HashKind { absence, time, sha1, sha256 }
+
+/// extract hash kind from line containing dub output
+auto extractHashKind(string str) {
+    import std.string : lineSplitter;
+
+    static enum link = ["time-dependent build":"time", "(sha1)":"sha1", "(sha256)":"sha256"];
+
+    foreach(line; str.lineSplitter)
+    {
+        foreach(e; link.byKeyValue) {
+            if (line.length >= e.key.length) {
+                if (line[$-e.key.length..$] == e.key)
+                    return e.value;
+            }
+        }
+    }
+
+    return "";
+}
+
+/// build target using given hash kind
+auto buildTargetUsing(HashKind kind, string[string] env = null) {
+    import std.exception : enforce;
+
+    auto dub = executeShell(buildNormalizedPath("..", "..", "bin", "dub") ~ 
+        " build --hash=%s".format(kind), env);
+    writeln("dub output:");
+    import std.string : lineSplitter;
+    foreach(line; dub.output.lineSplitter)
+        writeln("\t", line);
+    writeln("end of dub output");
+
+    enforce(dub.status == 0, "couldn't build the project, see above");
+
+    return dub.output;
+}
+
+/// check dub output to determine rebuild has not been triggered
+auto checkIfNoRebuild(string output) {
+    if (output.lineSplitter.any!(a=> a == "hash-dependent-build ~master: target for configuration \"application\" is up to date.")) {
+        writeln("\tOk. No rebuild triggered");
+        return true;
+    }
+    else
+        writeln("\tFail. Rebuild has been triggered");
+    return false;
+}
+
+/// check dub output to determine rebuild has been triggered
+auto checkIfRebuildTriggered(string output) {
+    if (output.lineSplitter.any!(a=> a == "hash-dependent-build ~master: building configuration \"application\"...")) {
+        writeln("Ok. Rebuild has been triggered");
+        return true;
+    }
+    else
+        writeln("Fail. No rebuild triggered");
+    return false;
+}
+
+int main()
+{
+    // delete old artifacts if any
+    const projectDir = buildPath(getcwd, "test", TestProjectName);
+    if (projectDir.exists)
+        projectDir.rmdirRecurse;
+    projectDir.mkdir;
+
+    chdir(projectDir);
+
+    // create test_project
+    {
+        auto dub = executeShell(buildNormalizedPath("..", "..", "bin", "dub") ~ " init --non-interactive");
+        if (dub.status != 0)
+        {
+            stderr.writeln("couldn't execute 'dub init test_project'");
+            stderr.writeln(dub.output);
+            return 1;
+        }
+    }
+
+    // build the project first time
+    writeln("\n---");
+    writeln("Build #1 (using hash dependent cache)");
+    writeln("Building the project from scratch");
+    writeln("Hash dependent build should be triggered");
+    auto output = buildTargetUsing(HashKind.sha256);
+    if (!checkIfRebuildTriggered(output))
+        return 1;
+
+    writeln("\n---");
+    writeln("Building #2 (using hash dependent cache)");
+    writeln("building the project that has been built (using hash dependent cache)");
+    writeln("Hash dependent build should NOT be triggered");
+    output = buildTargetUsing(HashKind.sha256);
+    if (!checkIfNoRebuild(output))
+        return 1;
+
+    // touch some source file(s)
+    {
+        SysTime atime, mtime;
+        const delay = dur!"msecs"(10);
+        getTimes(artifact_name, atime, mtime);
+        setTimes(source_name, atime + delay, mtime + delay);
+
+        // wait for the delay to avoid time related issues
+        import core.thread : Thread;
+        Thread.sleep(delay);
+    }
+
+    writeln("\n---");
+    writeln("Build #3 (using hash dependent cache)");
+    writeln("building the project that has been built (using hash dependent cache)");
+    writeln("but timestamp of source file(s) has been changed to be younger");
+    writeln("Hash dependent build should NOT be triggered");
+    output = buildTargetUsing(HashKind.sha256);
+    if (!checkIfNoRebuild(output))
+        return 1;
+
+    writeln("\n---");
+    writeln("build #4 (using time dependent cache)");
+    writeln("building the project that has been built (using hash dependent cache)");
+    writeln("but timestamp of source file(s) has been changed to be younger");
+    writeln("Time dependent build should be triggered");
+    output = buildTargetUsing(HashKind.time);
+    if (!checkIfRebuildTriggered(output))
+        return 1;
+
+    // edit some source file(s) preserving the file timestamp
+    {
+        SysTime atime, mtime;
+        getTimes(source_name, atime, mtime);
+
+        auto src = readText(source_name);
+        src ~= " ";
+        import std.file;
+        write(source_name, src);
+
+        setTimes(source_name, atime, mtime);
+    }
+
+    writeln("\n---");
+    writeln("build #5 (using time dependent cache)");
+    writeln("building the project that has been built (using both hash- and time- dependent cache)");
+    writeln("but source file(s) has been changed and timestamp of them was preserved");
+    writeln("Time dependent build should NOT be triggered");
+    output = buildTargetUsing(HashKind.time);
+    if (!checkIfNoRebuild(output))
+        return 1;
+
+    writeln("\n---");
+    writeln("build #6 (using hash dependent cache)");
+    writeln("building the project that has been built once (using both hash- and time- dependent cache)");
+    writeln("but source file(s) has been changed and timestamp of them was preserved");
+    writeln("Hash dependent build should be triggered");
+    output = buildTargetUsing(HashKind.sha256);
+    if (!checkIfRebuildTriggered(output))
+        return 1;
+
+    // Tests for command line interface option, environment variable and
+    // settings file values combination
+    {
+        string[string[3]] preset = [
+            // cli        env        settings
+            ["absence", "absence", "absence"]: "time",
+            ["absence", "absence", "sha1"   ]: "sha1",
+            ["absence", "sha256",  "sha1"   ]: "sha256",
+            ["absence", "sha1",    "sha256" ]: "sha1",
+            ["sha256",  "sha1",    "time"   ]: "sha256",
+            ["sha1",    "sha256",  "time"   ]: "sha1",
+            ["sha256",  "time",    "sha1"   ]: "sha256",
+            ["sha1",    "time",    "sha256" ]: "sha1",
+        ];
+        foreach(key, value; preset)
+        {
+            import std.conv : to, text;
+            import std.stdio : File, writefln;
+
+            writefln("cli: %s\tenv: %s\tsetting: %s\tresult: %s", key[0], key[1], key[2], value);
+
+            // write to settings file
+            {
+                File("./dub.settings.json", "w").writefln("{ \"hashKind\" : \"%s\" }", key[2]);
+            }
+
+            auto output = buildTargetUsing(key[0].to!HashKind, ["DUB_HASH_KIND":key[1]]);
+            auto str = extractHashKind(output);
+            assert(str == value, text("Given ", key, " but got `", str, "` instead of `", value, "`"));
+        }
+    }
+
+    // undo changes in source/app.d (i.e. restore its content)
+    // dub build --hash=sha256
+    // compare time of the artifact to previous value (the first value and the current one should be equal)
+
+    return 0;
+}

scripts/ci/travis.sh

@@ -28,6 +28,7 @@ if [ "$COVERAGE" = true ]; then
     bash codecov.sh
 else
     ./build.d
+    `pwd`/bin/dub --single scripts/ci/hash_dependent_build.d
     DUB=`pwd`/bin/dub DC=${DC} test/run-unittest.sh
 fi
 

source/dub/commandline.d

@@ -1083,6 +1083,7 @@ class GenerateCommand : PackageBuildCommand {
 		bool m_combined = false;
 		bool m_parallel = false;
 		bool m_printPlatform, m_printBuilds, m_printConfigs;
+		HashKind m_hash_kind;
 	}
 
 	this() @safe pure nothrow
@@ -1153,6 +1154,32 @@ class GenerateCommand : PackageBuildCommand {
 			logInfo("");
 		}
 
+		// Merge hash kind values got from command line option, settings file and environment
+		// Priority is:
+		// - cli option
+		// - environment
+		// - settings file
+		// - default value
+		{
+			import std.process : environment;
+			HashKind env_hash_kind;
+			string str;
+			try
+			{
+				str = environment.get("DUB_HASH_KIND");
+				if (str.length)
+					env_hash_kind = str.to!HashKind;
+			}
+			catch(Exception e)
+			{
+				logWarn("Wrong value of DUB_HASH_KIND environment variable: `%s`. Default value will be used", str);
+				env_hash_kind = HashKind.absence;
+			}
+
+			if (m_hash_kind == HashKind.absence) 
+				m_hash_kind = (env_hash_kind != HashKind.absence) ? env_hash_kind : dub.hashKind;
+		}
+
 		GeneratorSettings gensettings;
 		gensettings.platform = m_buildPlatform;
 		gensettings.config = m_buildConfig.length ? m_buildConfig : m_defaultConfig;
@@ -1169,6 +1196,7 @@ class GenerateCommand : PackageBuildCommand {
 		gensettings.tempBuild = m_tempBuild;
 		gensettings.parallelBuild = m_parallel;
 		gensettings.single = m_single;
+		gensettings.hashKind = m_hash_kind;
 
 		logDiagnostic("Generating using %s", m_generator);
 		dub.generateProject(m_generator, gensettings);
@@ -1207,6 +1235,9 @@ class BuildCommand : GenerateCommand {
 		args.getopt("n|non-interactive", &m_nonInteractive, [
 			"Don't enter interactive mode."
 		]);
+		args.getopt("hash", &m_hash_kind, [
+			"Use hash dependent build instead of file timestamp dependent one"
+		]);
 		super.prepare(args);
 		m_generator = "build";
 	}
@@ -1391,6 +1422,7 @@ class TestCommand : PackageBuildCommand {
 		settings.tempBuild = m_single;
 		settings.run = true;
 		settings.runArgs = app_args;
+		settings.hashKind = dub.hashKind;
 
 		dub.testProject(settings, m_buildConfig, NativePath(m_mainFile));
 		return 0;
@@ -2329,6 +2361,8 @@ class DustmiteCommand : PackageBuildCommand {
 			gensettings.compileCallback = check(m_compilerStatusCode, m_compilerRegex);
 			gensettings.linkCallback = check(m_linkerStatusCode, m_linkerRegex);
 			gensettings.runCallback = check(m_programStatusCode, m_programRegex);
+			gensettings.hashKind = dub.hashKind;
+
 			try dub.generateProject("build", gensettings);
 			catch (DustmiteMismatchException) {
 				logInfo("Dustmite test doesn't match.");

source/dub/dub.d

@@ -332,6 +332,19 @@ class Dub {
 	*/
 	@property string defaultArchitecture() const { return m_defaultArchitecture; }
 
+	/** Hash algorithm used for hashing cached build artefacts.
+
+		This is used by `BuildGenerator` to calculate hash sum of source files.
+	*/
+	@property HashKind hashKind() const { return m_config.hashKind; }
+
+	unittest
+	{
+		auto dub = new Dub();
+		dub.m_config = new DubConfig(Json(["hashKind": Json("sha1")]), null);
+		assert(dub.hashKind == HashKind.sha1);
+	}
+
 	/** Loads the package that resides within the configured `rootPath`.
 	*/
 	void loadPackage()
@@ -764,6 +777,7 @@ class Dub {
 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary, m_defaultArchitecture);
 		settings.buildType = "debug";
 		settings.run = true;
+		settings.hashKind = hashKind;
 
 		foreach (dependencyPackage; m_project.dependencies)
 		{
@@ -1259,6 +1273,7 @@ class Dub {
 		settings.buildType = "debug";
 		settings.run = true;
 		settings.runArgs = runArgs;
+		settings.hashKind = hashKind;
 		initSubPackage.recipe.buildSettings.workingDirectory = path.toNativeString();
 		template_dub.generateProject("build", settings);
 	}
@@ -1329,6 +1344,7 @@ class Dub {
 		settings.compiler = getCompiler(compiler_binary); // TODO: not using --compiler ???
 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary, m_defaultArchitecture);
 		settings.buildType = "debug";
+		settings.hashKind = hashKind;
 		settings.run = true;
 
 		auto filterargs = m_project.rootPackage.recipe.ddoxFilterArgs.dup;
@@ -1802,6 +1818,16 @@ private class DubConfig {
 		return SkipPackageSuppliers.none;
 	}
 
+	@property HashKind hashKind()
+	const {
+		if (auto pv = "hashKind" in m_data)
+			return (pv.get!string.to!HashKind);
+		if (m_parentConfig)
+			return m_parentConfig.hashKind;
+
+		return HashKind.absence;
+	}
+
 	@property NativePath[] customCachePaths()
 	{
 		import std.algorithm.iteration : map;