Incremental dexing + gradle 8.0 + kotlin 1.8.0

README.md

@@ -40,7 +40,7 @@ In case of succeful building a jar file named `compiled-mod-any-platform.jar` (t
 This file can be used both on android and on desktop.
 
 ## Building a desktop-only jar
-* Run `./gradlew jar`
+* Run `./gradlew jarDesktop`
 
 Everything is similar to the previous paragraph, except that the file will be named `compiled-mod-desktop.jar` and will __only be usable on desktop__.
 

build.gradle.kts

@@ -9,7 +9,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 plugins {
 	// when modifying the version of this plugin, you must also modify the version in other buildscripts:
 	// all modules must share the same version or the compilation will fail
-	kotlin("jvm") version "1.7.20-Beta"
+	kotlin("jvm") version "1.8.0"
 }
 
 allprojects {
@@ -31,16 +31,19 @@ allprojects {
 		// in addiction, never declare mindustry, arc and other mods (not libraries) as implementation deps!
 		// this will greatly increase the size of your mod and will most likely break something at runtime!
 
-		implementation(kotlin("stdlib-jdk8"))
 		// arc dependency
-		compileOnly("com.github.Anuken.Arc:arc-core:v137")
+		compileOnly("com.github.Anuken.Arc", "arc-core", "v141")
 		// mindustry dependency.
 		// jitpack refuses to compile mindustry due to its repository size, thus we're using a mirror
-		// 74a0321db8 is the hash of a commit, you can replace it with a different one or a specific version (e.g. v137)
-		compileOnly("com.github.Anuken:MindustryJitpack:74a0321db8")
+		compileOnly("com.github.Anuken", "MindustryJitpack", "v141")
 		//example of a library dependency. if you don't need it, remove this line.
 		//(note: this is not a mod, it's a library for mindustry mods, thus it should be added as an implementation dependency.
-		implementation("com.github.mnemotechnician:mkui:v1.1")
+		implementation("com.github.mnemotechnician", "mkui", "v1.2.1")
+	}
+
+	tasks.withType<JavaCompile> {
+		sourceCompatibility = "1.8"
+		targetCompatibility = "1.8"
 	}
 
 	tasks.withType<KotlinCompile> {

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.1-bin.zip
+networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -55,7 +55,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -80,10 +80,10 @@ do
     esac
 done
 
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
-
-APP_NAME="Gradle"
+# This is normally unused
+# shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
@@ -143,12 +143,16 @@ fi
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
+        # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
+        # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -205,6 +209,12 @@ set -- \
         org.gradle.wrapper.GradleWrapperMain \
         "$@"
 
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+    die "xargs is not available"
+fi
+
 # Use "xargs" to parse quoted args.
 #
 # With -n1 it outputs one arg per line, with the quotes and backslashes removed.

gradlew.bat

@@ -14,7 +14,7 @@
 @rem limitations under the License.
 @rem
 
-@if "%DEBUG%" == "" @echo off
+@if "%DEBUG%"=="" @echo off
 @rem ##########################################################################
 @rem
 @rem  Gradle startup script for Windows
@@ -25,7 +25,8 @@
 if "%OS%"=="Windows_NT" setlocal
 
 set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
+if "%DIRNAME%"=="" set DIRNAME=.
+@rem This is normally unused
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
@@ -40,7 +41,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto execute
+if %ERRORLEVEL% equ 0 goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -75,13 +76,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
 :end
 @rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
+if %ERRORLEVEL% equ 0 goto mainEnd
 
 :fail
 rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
 rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
+set EXIT_CODE=%ERRORLEVEL%
+if %EXIT_CODE% equ 0 set EXIT_CODE=1
+if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
+exit /b %EXIT_CODE%
 
 :mainEnd
 if "%OS%"=="Windows_NT" endlocal

mod-src/build.gradle.kts

@@ -1,10 +1,14 @@
-/*
+import java.io.OutputStream
+import java.security.MessageDigest
+import java.util.Base64
+
+/**
  * This file allows you to configure the compilation process of your mod.
  * There's several comments that you should read & follow in order to make everything work correctly
  */
 
 plugins {
-	kotlin("jvm") version "1.7.20-Beta"
+	kotlin("jvm") version "1.8.0"
 
 	/**
 	 * Uncomment this line and the "publications" block below if you want to publish to maven. 
@@ -40,11 +44,20 @@ publishing {
 		}
 	}
 }
-
 */
 
-/** Android-specific stuff. Do not modify unless you're 100% sure you know what you're doing! If you break this task, mobile users won't be able to use your mod!*/
+/** 
+ * Android-specific stuff. 
+ * Do not modify unless you're 100% sure you know what you're doing! 
+ * If you break this task, mobile users won't be able to use your mod!
+ */
 task("jarAndroid") {
+	fun hash(data: ByteArray): String =
+		MessageDigest.getInstance("MD5")
+		.digest(data)
+		.let { Base64.getEncoder().encodeToString(it) }
+		.replace('/', '_')
+
 	dependsOn("jar")
 
 	doLast {
@@ -78,17 +91,110 @@ task("jarAndroid") {
 			println("using ${platformRoot.absolutePath}")
 		}
 
-
 		//collect dependencies needed to translate java 8 bytecode code to android-compatible bytecode (yeah, android's dvm and art do be sucking)
-		val dependencies = (configurations.compileClasspath.get().files + configurations.runtimeClasspath.files + File(platformRoot, "android.jar")).map { it.path }
+		val dependencies = 
+			(configurations.runtimeClasspath.get().files)
+			.map { it.path }
+
+		val dexRoot = File("$buildDir/dex/").also { it.mkdirs() }
+		val dexCacheRoot = dexRoot.resolve("cache").also { it.mkdirs() }
+
+		// read the dex cache map (path-to-hash)
+		val dexCacheHashes = dexRoot.resolve("listing.txt")
+			.takeIf { it.exists() }
+			?.readText()
+			?.lineSequence()
+			?.map { it.split(" ") }
+			?.filter { it.size == 2 }
+			?.associate { it[0] to it[1] }
+			.orEmpty()
+			.toMutableMap()
+
+		// calculate hashes for all dependencies
+		val hashes = dependencies
+			.associate {
+				it to hash(File(it).readBytes())
+			}
+
+		// determime which dependencies can have their cached dex files reused and which can not
+		val reusable = ArrayList<String>()
+		val needReDex = HashMap<String, String>() // path-to-hash
+		hashes.forEach { (path, hash) ->
+			if (dexCacheHashes.getOrDefault(path, null) == hash) {
+				reusable += path
+			} else {
+				needReDex[path] = hash
+			}
+		}
+
+		println("${reusable.size} dependencies are already desugared and can be reused.")
+		if (needReDex.isNotEmpty()) println("Desugaring ${needReDex.size} dependencies.")
+
+		// for every non-reusable dependency, invoke d8 and save the new hash
+		var index = 1
+		needReDex.forEach { (dependency, hash) ->
+			println("Processing ${index++}/${needReDex.size} ($dependency)")
+
+			val outputDir = dexCacheRoot.resolve(hash(dependency.toByteArray())).also { it.mkdir() }
+			exec {
+				errorOutput = OutputStream.nullOutputStream()
+				commandLine(
+					"d8",
+					"--intermediate",
+					"--classpath", "${platformRoot.absolutePath}/android.jar",
+					"--min-api", "14", 
+					"--output", outputDir.absolutePath, 
+					dependency
+				)
+			}
+			println()
+			dexCacheHashes[dependency] = hash
+		}
+
+		// write the updated hash map to the file
+		dexCacheHashes.asSequence()
+			.map { (k, v) -> "$k $v" }
+			.joinToString("\n")
+			.let { dexRoot.resolve("listing.txt").writeText(it) }
+
+		if (needReDex.isNotEmpty()) println("Done.")
+		println("Preparing to desugar the project and merge dex files.")
+
+		val dexPathes = dependencies.map { 
+			dexCacheRoot.resolve(hash(it.toByteArray())).also { it.mkdir() }
+		}
+		// assemble the list of classpath arguments for project dexing
 		val dependenciesStr = Array<String>(dependencies.size * 2) {
-			if (it % 2 == 0) "--classpath" else dependencies.elementAt(it / 2)
+			if (it % 2 == 0) "--classpath" else dexPathes[it / 2].absolutePath
 		}
 
-		//dexing. As a result of this process, a .dex file will be added to the jar file. This requires d8 tool in your $PATH
+		// now, compile the project
 		exec {
-			workingDir("$buildDir/libs")
-			commandLine("d8", *dependenciesStr, "--min-api", "14", "--output", "${jarName}-android.jar", "${jarName}-desktop.jar")
+			val output = dexCacheRoot.resolve("project").also { it.mkdirs() }
+			commandLine(
+				"d8", 
+				*dependenciesStr,
+				"--classpath", "${platformRoot.absolutePath}/android.jar",
+				"--min-api", "14",
+				"--output", "$output",
+				"$buildDir/libs/$jarName.jar"
+			)
+		}
+
+		// finally, merge all dex files
+		exec {
+			val depDexes = dexPathes
+				.map { it.resolve("classes.dex") }.toTypedArray()
+				.filter { it.exists() } // some are empty
+				.map { it.absolutePath }
+				.toTypedArray()
+
+			commandLine(
+				"d8",
+				*depDexes,
+				dexCacheRoot.resolve("project/classes.dex").absolutePath,
+				"--output", "$buildDir/libs/$jarName-android.jar"
+			)
 		}
 	}
 }
@@ -98,25 +204,41 @@ task<Jar>("release") {
 	dependsOn("jarAndroid")
 
 	duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-	archiveFileName.set("${jarName}-any-platform.jar")
+	archiveFileName.set("$jarName-any-platform.jar")
 
 	from(
-		zipTree("$buildDir/libs/${jarName}-desktop.jar"),
+		zipTree("$buildDir/libs/${jarName}.jar"),
 		zipTree("$buildDir/libs/${jarName}-android.jar")
 	)
 
+	from(*configurations.runtimeClasspath.get().files.map { if (it.isDirectory()) it else zipTree(it) }.toTypedArray())
+
 	doLast {
-		delete { delete("$buildDir/libs/${jarName}-desktop.jar") }
+		delete { delete("$buildDir/libs/${jarName}.jar") }
 		delete { delete("$buildDir/libs/${jarName}-android.jar") }
 	}
 }
 
-
 tasks.jar {
+	duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+	archiveFileName.set("${jarName}.jar")
+
+	from(rootDir) {
+		include("mod.hjson")
+		include("icon.png")
+	}
+
+	from("../assets/") {
+		include("**")
+	}
+}
+
+/** Creates a desktop-only jar. */
+task<Jar>("jarDesktop") {
 	duplicatesStrategy = DuplicatesStrategy.EXCLUDE
 	archiveFileName.set("${jarName}-desktop.jar")
 
-	from(*configurations.runtimeClasspath.files.map { if (it.isDirectory()) it else zipTree(it) }.toTypedArray())
+	from(*configurations.runtimeClasspath.get().files.map { if (it.isDirectory()) it else zipTree(it) }.toTypedArray())
 
 	from(rootDir) {
 		include("mod.hjson")