From e62041408fa6a4e906089bc5bda81c60945e7346 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 18 Jan 2024 16:55:00 -0600 Subject: [PATCH] Reimplement populate_fiji.sh in Kotlin Co-authored-by: Giuseppe Barbieri --- build.gradle.kts | 15 +- .../kotlin/sciview/populateFiji.gradle.kts | 349 ++++++++++++++++++ populate_fiji.sh | 207 ----------- 3 files changed, 359 insertions(+), 212 deletions(-) create mode 100644 buildSrc/src/main/kotlin/sciview/populateFiji.gradle.kts delete mode 100755 populate_fiji.sh diff --git a/build.gradle.kts b/build.gradle.kts index 42c278d3a..101e0255e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -10,6 +10,7 @@ plugins { kotlin("kapt") sciview.publish sciview.sign + sciview.populateFiji id("org.jetbrains.dokka") jacoco `maven-publish` @@ -27,8 +28,6 @@ repositories { logger.warn("Using local Maven repository as source") mavenLocal() } - maven("https://jitpack.io") - mavenCentral() maven("https://maven.scijava.org/content/groups/public") } @@ -51,6 +50,10 @@ dependencies { version { strictly(sceneryVersion) } exclude("org.biojava.thirdparty", "forester") exclude("null", "unspecified") + + // from biojava artifacts; clashes with jakarta-activation-api + exclude("javax.xml.bind", "jaxb-api") + exclude("org.glassfish.jaxb", "jaxb-runtime") } implementation("net.java.dev.jna:jna-platform:5.11.0") @@ -61,8 +64,6 @@ dependencies { implementation("com.formdev:flatlaf") - implementation("org.slf4j:slf4j-simple") - // SciJava dependencies implementation("org.yaml:snakeyaml") { @@ -121,7 +122,7 @@ dependencies { implementation(platform(kotlin("bom"))) implementation(kotlin("stdlib-jdk8")) testImplementation(kotlin("test-junit")) - testImplementation("org.slf4j:slf4j-simple:1.7.36") + testImplementation("org.slf4j:slf4j-simple") implementation("sc.fiji:bigdataviewer-core") implementation("sc.fiji:bigdataviewer-vistools") @@ -445,4 +446,8 @@ jacoco { toolVersion = "0.8.11" } +task("copyDependencies", Copy::class) { + from(configurations.runtimeClasspath).into("$buildDir/dependencies") +} + java.withSourcesJar() diff --git a/buildSrc/src/main/kotlin/sciview/populateFiji.gradle.kts b/buildSrc/src/main/kotlin/sciview/populateFiji.gradle.kts new file mode 100644 index 000000000..898bb28aa --- /dev/null +++ b/buildSrc/src/main/kotlin/sciview/populateFiji.gradle.kts @@ -0,0 +1,349 @@ +package sciview + +import org.w3c.dom.Document +import org.w3c.dom.Element +import org.w3c.dom.NodeList +import org.xml.sax.InputSource +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.InputStreamReader +import java.net.URL +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.StandardCopyOption +import java.util.zip.GZIPInputStream +import javax.xml.parsers.DocumentBuilderFactory +import javax.xml.xpath.XPathConstants +import javax.xml.xpath.XPathFactory + +tasks { + register("populateFiji") { + dependsOn("jar") + doLast { + // Discern the path to the Fiji.app folder we are going to populate. + val fijiPath = System.getProperty("fiji.dir") ?: "Fiji.app" + logger.lifecycle("Populating ${fijiPath}...") + val fijiDir = File(fijiPath) + if (!fijiDir.isDirectory()) { + error("No such directory: ${fijiDir.absolutePath}") + } + + // Parse relevant update site databases. This information is useful + // for deciding which JAR files to copy, and which ones to leave alone. + val info = Info( + db("ImageJ", "https://update.imagej.net"), + db("Fiji", "https://update.fiji.sc"), + db("Java-8", "https://sites.imagej.net/Java-8"), + db("sciview", "https://sites.imagej.net/sciview-buttercup"), + mutableMapOf() + ) + + // Gather details of existing JAR files in the Fiji installation. + val jarsDir = fijiDir.resolve("jars") + project.fileTree(jarsDir).files.forEach { + if (it.extension == "jar") { + val relPath = it.path.substring(fijiDir.path.length) + info.localFiles.getOrPut(relPath.davce.acKey) { mutableListOf() } += it + } + } + + logger.info("--> Copying files into Fiji installation") + + // Copy sciview main artifact. + val mainJar = project.tasks.getByName("jar").outputs.files.singleFile + copy { + from(mainJar) + into(jarsDir) + eachFile { prepareToCopyFile(file, jarsDir, "jars", info) ?: exclude() } + } + + // Copy platform-independent JAR files. + copy { + from(configurations.named("runtimeClasspath")) + into(jarsDir) + eachFile { prepareToCopyFile(file, jarsDir, "jars", info) ?: exclude() } + } + + // Copy platform-specific JAR files. + for (platform in listOf("linux64", "macosx-arm64", "macosx", "win64")) { + val platformDir = jarsDir.resolve(platform) + copy { + from(configurations.named("runtimeClasspath")) + into(platformDir) + eachFile { prepareToCopyFile(file, platformDir, "jars/$platform", info) ?: exclude() } + } + } + + // HACK: Fix the naming of Fiji's renamed artifacts. + // This map is the inverse of the one called "renamedArtifacts" elsewhere. + val artifactsToFix = mapOf( + "jocl" to "org.jocl.jocl" + ) + for (file in project.fileTree(jarsDir).files) { + if (file.extension != "jar") continue + val relPath = file.path.substring(fijiDir.path.length) + val davce = relPath.davce + if (davce.classifier.isNotEmpty()) continue // DOUBLE HACK OMG + val badArtifactId = davce.artifactId + val goodArtifactId = artifactsToFix[badArtifactId] + if (goodArtifactId != null) { + val goodFile = File(file.path.replaceFirst(badArtifactId, goodArtifactId)) + logger.info("Renaming: $file -> $goodFile") + file.renameTo(goodFile) + } + } + + // Now that we populated Fiji, let's verify that it works. + logger.info("--> Testing installation") + val osName = System.getProperty("os.name") + val osArch = System.getProperty("os.arch") + val bits = if (osArch.contains("64")) "64" else "32" + val launcher = when { + osName == "Linux" -> "ImageJ-linux$bits" + osName == "Darwin" -> "Contents/MacOS/ImageJ-macosx" + osName.startsWith("Windows") -> "ImageJ-win$bits.exe" + else -> null + } + if (launcher == null) warn("Skipping test for unknown platform: $osName-$osArch") + else { + logger.info("Launcher executable = $launcher") + + val baos = ByteArrayOutputStream() + exec { + commandLine = listOf( + "$fijiDir/$launcher", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--", + "--headless", + "--run", + "sc.iview.commands.help.About" + ) + standardOutput = baos + } + val stdout = baos.toString().trim() + logger.info(stdout) + val success = stdout.startsWith("[INFO] SciView was created by Kyle Harrington") + if (!success) error("Test failed.") + logger.info("Test passed.") + } + } + } +} + +val String.dirname: String + get() { + val slash = lastIndexOf("/") + return if (slash < 0) "" else substring(0, slash) + } + +val String.extension: String + get() { + val dot = lastIndexOf(".") + return if (dot < 0) "" else substring(dot) + } + +/** Fixes backwards Windows path separators to use forward slashes instead. */ +val String.normalized: String + get() { + return replace("\\", "/") + } + +/** + * Parses a file path into five parts: containing directory, artifactId + * prefix, version string, classifier (if any), and file extension. + */ +val String.davce: DAVCE + get() { + val e = extension + val noExt = substring(0, length - e.length) + val d = noExt.normalized.dirname + val noDir = if (d.isEmpty()) noExt else noExt.substring(d.length + 1) + val cMatch = Regex(".*?-((native|windows|macos|linux).*)").matchEntire(noDir) + val c = cMatch?.groups?.get(1)?.value ?: "" + val noClass = if (c.isEmpty()) noDir else noDir.substring(0, noDir.length - c.length - 1) + val vMatch = Regex(".*?-([a-f0-9]{6}[a-f0-9]*|[0-9].*)").matchEntire(noClass) + val v = vMatch?.groups?.get(1)?.value ?: "" + val a = if (v.isEmpty()) noClass else noClass.substring(0, noClass.length - v.length - 1) + + // Do a sanity check. + var expected = if (d.isEmpty()) a else "$d/$a" + if (v.isNotEmpty()) expected += "-$v" + if (c.isNotEmpty()) expected += "-$c" + expected += e + if (expected != normalized) error("Failed to parse file path '$this' correctly (d=$d a=$a v=$v c=$c e=$e expected=$expected)") + + return DAVCE(d, a, v, c, e) + } + +infix fun String.compareVersions(version: String): Int { + val tokens1 = split(".") + val tokens2 = version.split(".") + for ((a, b) in tokens1.zip(tokens2)) { + try { + // Try to compare integers. + val aa = a.toLong() + val bb = b.toLong() + if (aa < bb) return -1 + if (aa > bb) return 1 + } + catch (e: NumberFormatException) { + // Fall back to comparing strings. + if (a < b) return -1 + if (a > b) return 1 + } + } + return tokens1.size.compareTo(tokens2.size) +} + +val Element.filename: String + get() { + return getAttribute("filename") ?: error("Plugin entry has no filename attribute") + } + +val Element.isObsolete: Boolean + get() { + for (j in 0 until childNodes.length) { + val child = childNodes.item(j) + if (child.nodeName == "version") return false + } + return true + } + +fun warn(message: String) { logger.warn("[WARNING] $message") } + +fun db(siteName: String, siteURL: String): Map { + val dbCacheDir = project.buildDir.resolve("tmp") + Files.createDirectories(dbCacheDir.toPath()) + val dbXml = dbCacheDir.resolve("$siteName.xml.gz") + val url = "$siteURL/db.xml.gz" + if (!dbXml.exists()) download(url, dbXml) + if (!dbXml.exists()) error("Download failed: $url") + + logger.info("--> Parsing ${dbXml.name}") + val db = parseGzippedXml(dbXml.readBytes()) + val xpath = XPathFactory.newInstance().newXPath() + val plugins = xpath.evaluate("//pluginRecords/plugin", db, XPathConstants.NODESET) as NodeList + val pluginMap = mutableMapOf() + for (i in 0 until plugins.length) { + val plugin = plugins.item(i) as Element + if (plugin.isObsolete) continue + if (!plugin.filename.endsWith(".jar")) continue + val ac = plugin.filename.davce.acKey + + val clashingPlugin = pluginMap[ac] + if (clashingPlugin != null) { + error("Clashing plugin entries: ${clashingPlugin.filename} vs. ${plugin.filename} " + + "(${clashingPlugin.filename.davce} vs. ${plugin.filename.davce})") + } + pluginMap[ac] = plugin + } + return pluginMap +} + +fun download(url: String, dest: File) { + logger.info("--> Downloading $url to $dest") + URL(url).openStream().use { inputStream -> + Files.copy(inputStream, dest.toPath(), StandardCopyOption.REPLACE_EXISTING) + } +} + +fun parseGzippedXml(gzippedXml: ByteArray): Document { + GZIPInputStream(ByteArrayInputStream(gzippedXml)).use { gzis -> + InputStreamReader(gzis, StandardCharsets.UTF_8).use { reader -> + val documentBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder() + return documentBuilder.parse(InputSource(reader)) + } + } +} + +data class AC(val artifactId: String, val classifier: String) { + val pathGuess: String get() { + val isWin = classifier.contains("win") + val isMac = classifier.contains("mac") + val isLinux = classifier.contains("linux") + val isArm64 = classifier.contains("arm64") + val subdir = when { + isLinux -> "linux64" + isMac && isArm64 -> "macosx-arm64" + isMac -> "macosx" + isWin -> "win64" + else -> return "jars" + } + return "jars/$subdir" + } +} +data class DAVCE( + val dirname: String, + val artifactId: String, + val version: String, + val classifier: String, + val extension: String +) { + val acKey: AC get() { + // HACK: Fiji has at least one artifact it renames. Bummer! + // Let's correct for that here, to ensure proper processing. + // This map is the inverse of the one called "artifactsToFix" elsewhere. + val renamedArtifacts = mapOf( + "org.jocl.jocl" to "jocl" + ) + return AC(renamedArtifacts[artifactId] ?: artifactId, classifier) + } +} +data class Info( + val imagej: Map, + val fiji: Map, + val java8: Map, + val sciview: Map, + val localFiles: MutableMap> +) + +/** + * Determines whether the given file belongs in the specified destination directory, and if so, + * does the necessary work to make preparations (e.g. delete other versions of the file). + * + * Returns true if the file is an appropriate fit for that directory and + * should be copied, or null if not. SMELL THE DELICIOUS CODE. + */ +fun prepareToCopyFile(file: File, destDir: File, pathPrefix: String, info: Info): Boolean? { + val davce = file.name.davce + val ac = davce.acKey + val corePlugin = info.java8[ac] ?: info.fiji[ac] ?: info.imagej[ac] + val sciviewPlugin = info.sciview[ac] + val plugin = sciviewPlugin ?: corePlugin + val pluginPath = plugin?.filename?.dirname ?: ac.pathGuess + if (pluginPath != pathPrefix) return null + + // FOUR CASES: + // * core only -- skip but do sanity checks + // * sciview only -- overwrite + // * core AND sciview -- overwrite but warn + // * neither -- overwrite but warn + + if (corePlugin != null && sciviewPlugin == null) { + // This file is provided by the core update site, not the sciview update site. + // We don't want to shadow any core files (which we aren't already shadowing). + logger.info("Skipping core file: ${file.name}") + // But if the version on the core update site is older, let's point that out. + val desiredVersion = davce.version + val coreVersion = corePlugin.filename.davce.version + if (desiredVersion compareVersions coreVersion > 0) { + warn("Ignoring file ${corePlugin.filename} from the core update site which " + + "is older than the desired dependency version ($desiredVersion > $coreVersion)") + } + return null + } + else { + // Delete existing versions of this file already in the Fiji.app installation. + for (localFile in info.localFiles[ac] ?: emptyList()) { + logger.info("Deleting: $localFile") + localFile.delete() + } + + // Log the impending copy operation, and warn of any weird situations. + logger.info("Copying: ${file.name} -> ${destDir.resolve(file.name)}") + if (corePlugin != null) warn("${file.name} SHADOWS the core update site!") + if (sciviewPlugin == null) warn("${file.name} is NEW to the sciview update site!") + } + return true +} diff --git a/populate_fiji.sh b/populate_fiji.sh deleted file mode 100755 index 38f1644b5..000000000 --- a/populate_fiji.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/bin/sh - -# -# populate_fiji.sh -# -# This script generates a local Fiji.app with SciView installed. - -# -- Functions -- - -left() { echo "${1%$2*}"; } -right() { echo "${1##*$2}"; } -mid() { right "$(left "$1" "$3")" "$2"; } - -die() { - echo "ERROR: $@" 1>&2; exit 1; -} - -# Copies the given Maven coordinate to the specified output directory. -install() { - (set -x; mvn dependency:copy -Dartifact="$1" -DoutputDirectory="$2" > /dev/null) || - die "Install failed" -} - -# Copies the given Maven coordinate to the specified output directory, keeping the groupId -installWithGroupId() { - (set -x; mvn dependency:copy -Dartifact="$1" -DoutputDirectory="$2" -Dmdep.prependGroupId=true > /dev/null) || - die "Install failed" -} - -# Deletes the natives JAR with the given artifactId and classifier. -deleteNative() { - (set -x; rm -f $FijiDirectory/jars/$1-[0-9]*-$2.jar $FijiDirectory/jars/*/$1-[0-9]*-$2.jar) || - die "Delete failed" -} - -# Deletes all natives JARs with the given artifactId. -deleteNatives() { - (set -x; rm -f $FijiDirectory/jars/$1-[0-9]*-natives-*.jar $FijiDirectory/jars/*/$1-[0-9]*-natives-*.jar) || - die "Delete failed" -} - -case "$(uname -s),$(uname -m)" in - Linux,x86_64) launcher=ImageJ-linux64 ;; - Linux,*) launcher=ImageJ-linux32 ;; - Darwin,*) launcher=Contents/MacOS/ImageJ-macosx ;; - MING*,*) launcher=ImageJ-win32.exe ;; - MSYS_NT*,*) launcher=ImageJ-win32.exe ;; - *) die "Unknown platform" ;; -esac - -# -- Check if we have a path given, in that case we do not download a new Fiji, but use the path given -- -if [ -z "$1" ] -then - echo "--> Installing into pristine Fiji installation" - echo "--> If you want to install into a pre-existing Fiji installation, run as" - echo " $0 path/to/Fiji.app" - # -- Determine correct ImageJ launcher executable -- - - - # -- Roll out a fresh Fiji -- - - if [ ! -f fiji-nojre.zip ] - then - echo - echo "--> Downloading Fiji" - curl -L -O https://downloads.imagej.net/fiji/latest/fiji-nojre.zip || - die "Could not download Fiji" - fi - - echo "--> Unpacking Fiji" - rm -rf Fiji.app - unzip fiji-nojre.zip || die "Could not unpack Fiji" - - echo - echo "--> Updating Fiji" - Fiji.app/$launcher --update update-force-pristine - FijiDirectory=Fiji.app -else - echo "--> Installing into Fiji installation at $1" - FijiDirectory=$1 -fi - -echo -echo "--> Copying dependencies into Fiji installation" -(set -x; mvn -Dscijava.app.directory=$FijiDirectory) || - die "Failed to copy dependencies into Fiji directory" - -echo "--> Removing slf4j bindings" -(set -x; rm -f $FijiDirectory/jars/slf4j-simple-*.jar) || - die "Failed to remove slf4j bindings" - -# -- Put back jar/gluegen-rt and jar/jogl-all -- -echo -echo "--> Reinstalling gluegen-rt, jogl-all, jocl, jinput, and ffmpeg" -gluegenJar=$(echo $FijiDirectory/jars/gluegen-rt-main-*.jar) -gluegenVersion=$(mid "$gluegenJar" "-" ".jar") -install "org.jogamp.gluegen:gluegen-rt:$gluegenVersion" $FijiDirectory/jars - -joglJar=$(echo $FijiDirectory/jars/jogl-all-main-*.jar) -joglVersion=$(mid "$joglJar" "-" ".jar") -install "org.jogamp.jogl:jogl-all:$joglVersion" $FijiDirectory/jars - -joclGAV=$(mvn dependency:tree | grep jocl | awk -e '{print $NF}' | cut -d: -f1-4 | sed 's/:jar//g') -installWithGroupId "$joclGAV" $FijiDirectory/jars - -jinputGAV=$(mvn dependency:tree | grep jinput | head -n1 | awk -e '{print $NF}' | cut -d: -f1-4 | sed 's/:jar//g' | sed 's/:compile//g') -install "$jinputGAV" $FijiDirectory/jars -installWithGroupId "$jinputGAV:jar:natives-all" $FijiDirectory/jars/win64 -installWithGroupId "$jinputGAV:jar:natives-all" $FijiDirectory/jars/linux64 -installWithGroupId "$jinputGAV:jar:natives-all" $FijiDirectory/jars/macosx -echo "--> Removing jinput natives from JAR root" -(set -x; rm -f $FijiDirectory/jars/jinput-*-natives-all.jar) - -ffmpegGAV=$(mvn dependency:tree | grep 'ffmpeg:jar' | head -n1 | awk -e '{print $NF}' | cut -d: -f1-4 | sed 's/:jar//g' | sed 's/:compile//g') -installWithGroupId "$ffmpegGAV" $FijiDirectory/jars -installWithGroupId "$ffmpegGAV:jar:windows-x86_64" $FijiDirectory/jars/win64 -installWithGroupId "$ffmpegGAV:jar:linux-x86_64" $FijiDirectory/jars/linux64 -installWithGroupId "$ffmpegGAV:jar:macosx-x86_64" $FijiDirectory/jars/macosx - -# -- Get the latest imagej-launcher -- [CHECK IF THIS CAN BE REMOVED] - -wget "https://maven.scijava.org/service/local/repositories/releases/content/net/imagej/imagej-launcher/5.0.2/imagej-launcher-5.0.2-linux64.exe" -O $FijiDirectory/ImageJ-linux64 || - die "Could not get linux64 launcher" -chmod +x $FijiDirectory/ImageJ-linux64 -mkdir -p $FijiDirectory/Contents/MacOS/ -wget "https://maven.scijava.org/service/local/repositories/releases/content/net/imagej/imagej-launcher/5.0.2/imagej-launcher-5.0.2-macosx.exe" -O $FijiDirectory/Contents/MacOS/ImageJ-macosx || - die "Could not get macOS launcher" -chmod +x $FijiDirectory/Contents/MacOS/ImageJ-macosx -wget "https://maven.scijava.org/service/local/repositories/releases/content/net/imagej/imagej-launcher/5.0.2/imagej-launcher-5.0.2-win32.exe" -O $FijiDirectory/ImageJ-win32 || - die "Could not get Win32 launcher" -chmod +x $FijiDirectory/ImageJ-win32 -wget "https://maven.scijava.org/service/local/repositories/releases/content/net/imagej/imagej-launcher/5.0.2/imagej-launcher-5.0.2-win64.exe" -O $FijiDirectory/ImageJ-win64 || - die "Could not get Win64 launcher" -chmod +x $FijiDirectory/ImageJ-win64 - -# -- Fix old miglayout - -rm $FijiDirectory/jars/miglayout-3.7.4-swing.jar -install "com.miglayout:miglayout-swing:5.2" $FijiDirectory/jars - -# -- Fix imagej-mesh versions with jitpack version clashing (temporary) - -rm $FijiDirectory/jars/imagej-mesh-* -install "net.imagej:imagej-mesh:0.8.1" $FijiDirectory/jars - -# -- Get the list of native libraries -- - -# [NB] dependency:list emits G:A:P:C:V but dependency:copy needs G:A:V:P:C. -echo -echo "--> Extracting list of native dependencies" -natives=$(mvn -B dependency:list | - grep natives | - sed -e 's/^\[INFO\] *\([^:]*\):\([^:]*\):\([^:]*\):\([^:]*\):\([^:]*\):.*/\1:\2:\5:\3:\4/' | - grep -v -- '-\(android\|armv6\|solaris\)' | - sort) -for gavpc in $natives -do - gavp=$(left "$gavpc" ':') - gav=$(left "$gavp" ':') - ga=$(left "$gav" ':') - g=$(left "$ga" ':') - a=$(right "$ga" ':') - v=$(right "$gav" ':') - p=$(right "$gavp" ':') - c=$(right "$gavpc" ':') - echo - echo "[$a-$v-$c]" - case "$g" in - org.lwjgl|graphics.scenery) - deleteNatives "$a" - # [NB] Install all architectures manually; only one is a dependency. - install "$gavp:natives-windows" $FijiDirectory/jars/win64 - install "$gavp:natives-macos" $FijiDirectory/jars/macosx - install "$gavp:natives-linux" $FijiDirectory/jars/linux64 - ;; - *) - deleteNative "$a" "$c" - case "$c" in - natives-win*-i586) continue ;; - natives-win*) platform=win64 ;; - natives-linux*-i586) continue ;; - natives-linux*) platform=linux64 ;; - natives-osx|natives-mac*) platform=macosx ;; - natives-all*) continue ;; - *) die "Unsupported platform: $c" ;; - esac - install "$gavpc" "$FijiDirectory/jars/$platform" - ;; - esac -done - -# -- Now that we populated fiji, let's double check that it works -- - -echo -echo "--> Testing installation with command: sc.iview.commands.help.About" -EXE="$FijiDirectory//$launcher" -OUT_TEST=$($EXE --headless --run sc.iview.commands.help.About) -echo $OUT_TEST - -if [ -z "$OUT_TEST" ] -then - echo "Test failed" - exit 1 -else - echo "Test passed" -fi -