diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 9ffabee..d1c7084 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -23,7 +23,7 @@ jobs: - name: Run gradle build and publish run: > - gradle distZip + gradle shadowDistZip -PmineinabyssMavenUsername=${{ secrets.MAVEN_PUBLISH_USERNAME }} -PmineinabyssMavenPassword=${{ secrets.MAVEN_PUBLISH_PASSWORD }} - name: Get version from gradle @@ -40,4 +40,4 @@ jobs: prerelease: false automatic_release_tag: v${{ steps.extract_version.outputs.version }} files: | - ./build/distributions/keepup*.zip + ./build/distributions/keepup-shadow*.zip diff --git a/build.gradle.kts b/build.gradle.kts index 8623a31..643d849 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,8 +1,9 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { - kotlin("jvm") version "1.9.20" - kotlin("plugin.serialization") version "1.9.20" + kotlin("jvm") version "1.9.21" + kotlin("plugin.serialization") version "1.9.21" + id("com.github.johnrengelman.shadow") version "8.1.1" application } @@ -24,7 +25,12 @@ dependencies { implementation("com.github.ajalt.clikt:clikt:3.5.0") implementation("com.lordcodes.turtle:turtle:0.8.0") implementation("com.jayway.jsonpath:json-path:2.7.0") + implementation("io.ktor:ktor-client-core:2.3.8") + implementation("io.ktor:ktor-client-cio:2.3.8") + implementation("com.github.ajalt.mordant:mordant:2.3.0") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0-RC2") implementation("com.sealwu:kscript-tools:1.0.22") + implementation("io.ktor:ktor-client-cio-jvm:2.3.8") testImplementation(kotlin("test")) } @@ -35,3 +41,13 @@ tasks.test { application { mainClass.set("MainKt") } + +kotlin { + jvmToolchain(17) +} + +tasks { + shadowJar { + minimize() + } +} diff --git a/gradle.properties b/gradle.properties index 1e90295..5750944 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,2 @@ kotlin.code.style=official -version=1.2.3 +version=2.0.0-beta.1 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e411586..a595206 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/kotlin/GithubDownload.kt b/src/main/kotlin/GithubDownload.kt deleted file mode 100644 index 577ad45..0000000 --- a/src/main/kotlin/GithubDownload.kt +++ /dev/null @@ -1,52 +0,0 @@ -import commands.CachedCommand -import commands.DownloadedItem -import commands.Wget -import helpers.GithubReleaseOverride -import java.nio.file.Path -import kotlin.io.path.div - -/** - * Ex url "github:MineInAbyss/Idofront:v0.20.6:*.jar" - */ -context(Keepup) -class GithubDownload(val repo: String, val releaseVersion: String, val artifactRegexString: String) { - val artifactRegex = artifactRegexString.toRegex() - - companion object { - context(Keepup) - fun from(string: String): GithubDownload { - val (repo, release, grep) = string.removePrefix("github:").split(":") - return GithubDownload(repo, release, grep) - } - } - - fun download(targetDir: Path, forceLatest: GithubReleaseOverride): List { - val version = when (forceLatest) { - GithubReleaseOverride.LATEST_RELEASE -> "latest-release" - GithubReleaseOverride.LATEST -> "latest" - else -> releaseVersion - } - val releaseURL = if (version == "latest") "latest" else "tags/$releaseVersion" - val downloadURLs = CachedCommand( - buildString { - append("curl -s ") - if (githubAuthToken != null) - append("-H \"Authorization: token $githubAuthToken\" ") - if (overrideGithubRelease == GithubReleaseOverride.LATEST) - append("https://api.github.com/repos/$repo/releases | jq 'map(select(.draft == false)) | sort_by(.published_at) | last'") - else - append("https://api.github.com/repos/$repo/releases/$releaseURL") - append(" | grep 'browser_download_url'") - }, - targetDir / "response-${repo.replace("/", "-")}-$version", - expiration = cacheExpirationTime.takeIf { version == "latest" } - ).getFromCacheOrEval().split("\n") - .map { it.trim().removePrefix("\"browser_download_url\": \"").trim('"') } - .filter { it.contains(artifactRegex) } - - println("Got URLs github:$repo:$version:$artifactRegexString $downloadURLs") - - return downloadURLs - .mapNotNull { Wget(it, targetDir, updateIfChangedOnServer = forceLatest == GithubReleaseOverride.LATEST) } - } -} diff --git a/src/main/kotlin/Main.kt b/src/main/kotlin/Main.kt index 9f72a5c..8e27e4a 100644 --- a/src/main/kotlin/Main.kt +++ b/src/main/kotlin/Main.kt @@ -11,18 +11,39 @@ import com.github.ajalt.clikt.parameters.types.choice import com.github.ajalt.clikt.parameters.types.enum import com.github.ajalt.clikt.parameters.types.inputStream import com.github.ajalt.clikt.parameters.types.path +import com.github.ajalt.mordant.animation.progressAnimation +import com.github.ajalt.mordant.rendering.TextColors.brightGreen +import com.github.ajalt.mordant.rendering.TextColors.yellow +import com.github.ajalt.mordant.terminal.Terminal import com.jayway.jsonpath.JsonPath +import config.GithubConfig +import downloading.DownloadParser +import downloading.DownloadResult +import downloading.Source import helpers.* -import kotlin.io.path.* +import io.ktor.client.* +import io.ktor.client.engine.cio.* +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlin.io.path.absolute +import kotlin.io.path.createDirectories +import kotlin.io.path.div import kotlin.time.Duration import kotlin.time.Duration.Companion.minutes +val keepup by lazy { Keepup() } +val t by lazy { Terminal() } + class Keepup : CliktCommand() { init { context { autoEnvvarPrefix = "KEEPUP" } } + // === Arguments === val input by argument(help = "Path to the file").inputStream() @@ -43,6 +64,9 @@ class Keepup : CliktCommand() { val ignoreSimilar by option(help = "Don't create symlinks for files with matching characters before the first number") .flag(default = true) + val failAllDownloads by option(help = "Don't actually download anything, useful for testing") + .flag(default = false) + val overrideGithubRelease by option(help = "Force downloading the latest version of files from GitHub") .enum() .default(GithubReleaseOverride.NONE) @@ -53,38 +77,75 @@ class Keepup : CliktCommand() { val githubAuthToken: String? by option(help = "Used to access private repos or get a higher rate limit") + val githubConfig by lazy { + GithubConfig( + githubAuthToken = githubAuthToken, + overrideGithubRelease = overrideGithubRelease, + cacheExpirationTime = cacheExpirationTime, + ) + } + + val progress = t.progressAnimation { + text("Keepup!") + percentage() + progressBar() + completed() + timeRemaining() + } override fun run() { if (overrideGithubRelease != GithubReleaseOverride.NONE) - echo("Overriding GitHub release versions to $overrideGithubRelease") + t.println("${yellow("[!]")} Overriding GitHub release versions to $overrideGithubRelease") val jsonInput = if (fileType == "hocon") { - println("Converting HOCON to JSON") + t.println("Converting HOCON to JSON") renderHocon(input) } else input - println("Parsing input") + t.println("Parsing input") val parsed = JsonPath.parse(jsonInput) val items = parsed.read>(jsonPath) val strings = getLeafStrings(items) - println("Clearing symlinks") + t.println("Clearing symlinks") clearSymlinks(dest) - - println("Creating new symlinks") - strings.forEach { (key, source) -> - val isolatedPath = (downloadPath / key).absolute() - isolatedPath.createDirectories() - val files = dest.listDirectoryEntries().filter { it.isRegularFile() } - download(source, isolatedPath).forEach download@{ item -> - if (ignoreSimilar && files.any { similar(item.name, it.name) }) { - println("Skipping ${item.name} because it is similar to an existing file") - return@download + progress.updateTotal(strings.size.toLong()) + progress.start() + + runBlocking(Dispatchers.IO) { + val channel = Channel() + launch { + HttpClient(CIO).use { client -> + val similarFileChecker = if (ignoreSimilar) SimilarFileChecker(dest) else null + val downloader = DownloadParser( + client, + githubConfig, + similarFileChecker + ) + + strings.map { (key, downloadQuery) -> + val downloadPathForKey = (downloadPath / key).absolute() + downloadPathForKey.createDirectories() + launch { + downloader.download(Source(key, downloadQuery), downloadPathForKey) + .forEach { channel.send(it) } + progress.advance(1) + } + }.joinAll() } - linkToDest(dest, isolatedPath, item) + channel.close() } + for (result in channel) { + if (result is DownloadResult.HasFiles) { + linkToDest(dest, result) + } + result.printToConsole() + } + + progress.clear() + progress.stop() + t.println(brightGreen("Keepup done!")) } - println("Keepup done!") } } -fun main(args: Array) = Keepup().main(args) +fun main(args: Array) = keepup.main(args) diff --git a/src/main/kotlin/SimilarFileChecker.kt b/src/main/kotlin/SimilarFileChecker.kt new file mode 100644 index 0000000..bffa264 --- /dev/null +++ b/src/main/kotlin/SimilarFileChecker.kt @@ -0,0 +1,35 @@ +import downloading.DownloadResult +import java.nio.file.Path +import kotlin.io.path.isRegularFile +import kotlin.io.path.listDirectoryEntries +import kotlin.io.path.name + +class SimilarFileChecker( + val dest: Path +) { + val names = dest + .listDirectoryEntries() + .filter { it.isRegularFile() } + .map { it.name } + + fun findSimilarFileTo(key: String): String? { + return names.firstOrNull { similar(it, key) } + } + + fun filterSimilarFiles(results: List): List { + return results.map { + if (it is DownloadResult.HasFiles) { + val similarFile = this.findSimilarFileTo(it.file.name) + if (similarFile != null) + return@map DownloadResult.SkippedBecauseSimilar(it.keyInConfig, similarFile) + } + it + } + } + + /** Removes everything between the first digit and ext of the file */ + fun String.removeVersion() = "${takeWhile { !it.isDigit() }}.${takeLastWhile { it != '.' }}" + + /** Checks if two strings are similar with their versions removed */ + fun similar(a: String, b: String): Boolean = a.removeVersion() == b.removeVersion() +} diff --git a/src/main/kotlin/commands/CachedCommand.kt b/src/main/kotlin/commands/CachedCommand.kt index e5b1233..d54276f 100644 --- a/src/main/kotlin/commands/CachedCommand.kt +++ b/src/main/kotlin/commands/CachedCommand.kt @@ -1,7 +1,6 @@ package commands import evalBash -import java.nio.file.LinkOption import java.nio.file.Path import java.time.LocalDateTime import kotlin.io.path.* @@ -9,17 +8,26 @@ import kotlin.time.Duration import kotlin.time.toJavaDuration class CachedCommand(val command: String, val path: Path, val expiration: Duration? = null) { - fun getFromCacheOrEval(): String { + class Result( + val wasCached: Boolean, + val result: String, + ) + + fun getFromCacheOrEval(): Result { val expirationPath = path.parent / (path.name + ".expiration") // get current time val time = LocalDateTime.now() val expiryDate = expirationPath.takeIf { it.exists() }?.readText()?.let { LocalDateTime.parse(it) } if (path.exists() && (expiryDate == null || time < expiryDate)) { - return path.readText() + return Result(true, path.readText()) } - val evaluated = command.evalBash(env = mapOf()).getOrThrow() + val evaluated = command.evalBash(env = mapOf()) + .onFailure { + throw IllegalStateException("Failed to evaluate command $command, result:\n${this.stderr.joinToString("\n")}") + } + .getOrThrow() path.deleteIfExists() path.createParentDirectories().createFile().writeText(evaluated) // write expiration date @@ -27,6 +35,6 @@ class CachedCommand(val command: String, val path: Path, val expiration: Duratio expirationPath.deleteIfExists() expirationPath.createFile().writeText((time + expiration.toJavaDuration()).toString()) } - return evaluated + return Result(false, evaluated) } } diff --git a/src/main/kotlin/commands/DownloadedItem.kt b/src/main/kotlin/commands/DownloadedItem.kt deleted file mode 100644 index 3732557..0000000 --- a/src/main/kotlin/commands/DownloadedItem.kt +++ /dev/null @@ -1,12 +0,0 @@ -package commands - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable - -@Serializable -class DownloadedItem( - @SerialName("Path") - val relativePath: String, - @SerialName("Name") - val name: String, -) diff --git a/src/main/kotlin/commands/Rclone.kt b/src/main/kotlin/commands/Rclone.kt index 2d9d5c3..1b2a992 100644 --- a/src/main/kotlin/commands/Rclone.kt +++ b/src/main/kotlin/commands/Rclone.kt @@ -1,33 +1,17 @@ package commands -import com.lordcodes.turtle.ShellRunException import com.lordcodes.turtle.shellRun -import kotlinx.serialization.json.Json import java.nio.file.Path +import kotlin.io.path.div object Rclone { val rclone = "rclone" - private val json = Json { ignoreUnknownKeys = true } - fun sync(source: String, targetDir: Path): DownloadedItem? { - return runCatching { - shellRun(rclone, listOf("sync", source, targetDir.toString())) - } - .map { - val name = source.substringAfterLast("/") - DownloadedItem("$targetDir/$name", name) - } - .onSuccess { println("Downloaded $source") } - .onFailure { - if (it is ShellRunException) System.err.println( - "Error downloading $source\n${ - it.message?.prependIndent( - " " - ) - }" - ) - } - .getOrNull() + fun sync(source: String, targetDir: Path): Path { + shellRun(rclone, listOf("sync", source, targetDir.toString())) + // TODO support multiple item downloads via rclone + val name = source.substringAfterLast("/") + return targetDir / name } } diff --git a/src/main/kotlin/commands/Wget.kt b/src/main/kotlin/commands/Wget.kt deleted file mode 100644 index 6eb7229..0000000 --- a/src/main/kotlin/commands/Wget.kt +++ /dev/null @@ -1,36 +0,0 @@ -package commands - -import com.lordcodes.turtle.ShellRunException -import com.lordcodes.turtle.shellRun -import java.nio.file.Path - -object Wget { - operator fun invoke(url: String, targetDir: Path, updateIfChangedOnServer: Boolean = false): DownloadedItem? { - val result = runCatching { - shellRun( - "wget", - listOf( - if (updateIfChangedOnServer) "--timestamping" else "--no-clobber", - url, - "-P", - targetDir.toString() - ) - ) - } - return result.map { - val name = url.substringAfterLast("/") - DownloadedItem("$targetDir/$name", name) - } - .onSuccess { println("Downloaded $url") } - .onFailure { - if (it is ShellRunException) System.err.println( - "Error downloading $url\n${ - it.message?.prependIndent( - " " - ) - }" - ) - } - .getOrNull() - } -} diff --git a/src/main/kotlin/config/GithubConfig.kt b/src/main/kotlin/config/GithubConfig.kt new file mode 100644 index 0000000..80a0c43 --- /dev/null +++ b/src/main/kotlin/config/GithubConfig.kt @@ -0,0 +1,10 @@ +package config + +import helpers.GithubReleaseOverride +import kotlin.time.Duration + +class GithubConfig( + val overrideGithubRelease: GithubReleaseOverride, + val githubAuthToken: String?, + val cacheExpirationTime: Duration?, +) diff --git a/src/main/kotlin/downloading/DownloadParser.kt b/src/main/kotlin/downloading/DownloadParser.kt new file mode 100644 index 0000000..bef35e2 --- /dev/null +++ b/src/main/kotlin/downloading/DownloadParser.kt @@ -0,0 +1,61 @@ +package downloading + +import SimilarFileChecker +import com.lordcodes.turtle.ShellRunException +import config.GithubConfig +import downloading.github.GithubArtifact +import downloading.github.GithubDownload +import io.ktor.client.* +import keepup +import kotlinx.coroutines.delay +import java.nio.file.Path +import kotlin.random.Random +import kotlin.time.Duration.Companion.milliseconds + +class DownloadParser( + val client: HttpClient, + val githubConfig: GithubConfig, + val similarFileChecker: SimilarFileChecker?, +) { + val httpRegex = "^https?://.*".toRegex() + + /** + * Downloads a file from a [source] definition into a [targetDir] + * + * @param source Takes form of an https url, rclone remote, or `.` to ignore + */ + suspend fun download(source: Source, targetDir: Path): List { + if (keepup.failAllDownloads) { + delay(Random.nextLong(500, 2000).milliseconds) + return listOf(DownloadResult.Failure("Testing flag enabled", source.keyInConfig)) + } + + val downloader: Downloader = when { + source.query == "." -> return emptyList() + source.query.startsWith("github:") -> GithubDownload( + client = client, + config = githubConfig, + artifact = GithubArtifact.from(source), + targetDir = targetDir, + ) + + source.query.matches(httpRegex) -> HttpDownload(client, source, targetDir) + else -> RcloneDownload(source, targetDir) + } + + val results = runCatching { + downloader.download() + }.getOrElse { error -> + val message = (error.message).takeIf { error is ShellRunException } ?: error.stackTraceToString() + + listOf( + DownloadResult.Failure( + message = "Program errored,\n${message.prependIndent("\t")}", + keyInConfig = source.keyInConfig, + ) + ) + } + + return similarFileChecker?.filterSimilarFiles(results) ?: results + } +} diff --git a/src/main/kotlin/downloading/DownloadResult.kt b/src/main/kotlin/downloading/DownloadResult.kt new file mode 100644 index 0000000..0274dc8 --- /dev/null +++ b/src/main/kotlin/downloading/DownloadResult.kt @@ -0,0 +1,32 @@ +package downloading + +import java.nio.file.Path + +sealed interface DownloadResult { + val keyInConfig: String + + sealed interface HasFiles : DownloadResult { + val file: Path + } + + data class SkippedBecauseSimilar( + override val keyInConfig: String, + val similarTo: String, + ) : DownloadResult + + data class SkippedBecauseCached( + override val file: Path, + override val keyInConfig: String, + ) : HasFiles + + data class Downloaded( + override val file: Path, + override val keyInConfig: String, + val overrideInfoMsg: String? = null, + ) : HasFiles + + data class Failure( + val message: String, + override val keyInConfig: String, + ) : DownloadResult +} diff --git a/src/main/kotlin/downloading/Downloader.kt b/src/main/kotlin/downloading/Downloader.kt new file mode 100644 index 0000000..4065d8c --- /dev/null +++ b/src/main/kotlin/downloading/Downloader.kt @@ -0,0 +1,5 @@ +package downloading + +interface Downloader { + suspend fun download(): List +} diff --git a/src/main/kotlin/downloading/HttpDownload.kt b/src/main/kotlin/downloading/HttpDownload.kt new file mode 100644 index 0000000..5ea7312 --- /dev/null +++ b/src/main/kotlin/downloading/HttpDownload.kt @@ -0,0 +1,38 @@ +package downloading + +import io.ktor.client.* +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.http.* +import io.ktor.util.cio.* +import io.ktor.utils.io.* +import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.io.path.readText +import kotlin.io.path.writeText + +class HttpDownload( + val client: HttpClient, + val source: Source, + val targetDir: Path, + val fileName: String = source.query.substringAfterLast("/"), +) : Downloader { + override suspend fun download(): List { + val cacheFile = targetDir.resolve("$fileName.cache") + val targetFile = targetDir.resolve(fileName) + val headers = client.head(source.query).headers + val lastModified = headers["Last-Modified"]?.fromHttpToGmtDate() + val length = headers["Content-Length"]?.toLongOrNull() + + val cache = "Last-Modified: $lastModified, Content-Length: $length" + if (targetFile.exists() && cacheFile.readText() == cache) + return listOf(DownloadResult.SkippedBecauseCached(targetFile, source.keyInConfig)) + cacheFile.writeText(cache) + + client.get(source.query) + .bodyAsChannel() + .copyAndClose(targetFile.toFile().writeChannel()) + + return listOf(DownloadResult.Downloaded(targetFile, source.keyInConfig)) + } +} diff --git a/src/main/kotlin/downloading/RcloneDownload.kt b/src/main/kotlin/downloading/RcloneDownload.kt new file mode 100644 index 0000000..2822d1f --- /dev/null +++ b/src/main/kotlin/downloading/RcloneDownload.kt @@ -0,0 +1,15 @@ +package downloading + +import commands.Rclone +import helpers.MSG +import java.nio.file.Path + +class RcloneDownload( + val source: Source, + val targetDir: Path, +) : Downloader { + override suspend fun download(): List { + val downloadPath = Rclone.sync(source.query, targetDir) + return listOf(DownloadResult.Downloaded(downloadPath, source.query, overrideInfoMsg = MSG.rclone)) + } +} diff --git a/src/main/kotlin/downloading/Source.kt b/src/main/kotlin/downloading/Source.kt new file mode 100644 index 0000000..36455e1 --- /dev/null +++ b/src/main/kotlin/downloading/Source.kt @@ -0,0 +1,6 @@ +package downloading + +data class Source( + val keyInConfig: String, + val query: String, +) diff --git a/src/main/kotlin/downloading/github/GithubArtifact.kt b/src/main/kotlin/downloading/github/GithubArtifact.kt new file mode 100644 index 0000000..293e982 --- /dev/null +++ b/src/main/kotlin/downloading/github/GithubArtifact.kt @@ -0,0 +1,19 @@ +package downloading.github + +import downloading.Source + +data class GithubArtifact( + val source: Source, + val repo: String, + val releaseVersion: String, + val regex: String, +) { + val calculatedRegex = regex.toRegex() + + companion object { + fun from(source: Source): GithubArtifact { + val (repo, release, grep) = source.query.removePrefix("github:").split(":") + return GithubArtifact(source, repo, release, grep) + } + } +} diff --git a/src/main/kotlin/downloading/github/GithubDownload.kt b/src/main/kotlin/downloading/github/GithubDownload.kt new file mode 100644 index 0000000..ccd4d41 --- /dev/null +++ b/src/main/kotlin/downloading/github/GithubDownload.kt @@ -0,0 +1,72 @@ +package downloading.github + +import com.github.ajalt.mordant.rendering.TextColors +import com.github.ajalt.mordant.rendering.TextColors.gray +import commands.CachedCommand +import config.GithubConfig +import downloading.DownloadResult +import downloading.Downloader +import downloading.HttpDownload +import downloading.Source +import helpers.GithubReleaseOverride +import helpers.MSG +import io.ktor.client.* +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import t +import java.nio.file.Path +import kotlin.io.path.div + +/** + * Ex url "github:MineInAbyss/Idofront:v0.20.6:*.jar" + */ +class GithubDownload( + val client: HttpClient, + val config: GithubConfig, + val artifact: GithubArtifact, + val targetDir: Path, +) : Downloader { + + override suspend fun download(): List { + val version = when (config.overrideGithubRelease) { + GithubReleaseOverride.LATEST_RELEASE -> "latest-release" + GithubReleaseOverride.LATEST -> "latest" + else -> artifact.releaseVersion + } + val releaseURL = if (version == "latest") "latest" else "tags/${artifact.releaseVersion}" + + //TODO convert to ktor + val commandResult = CachedCommand( + buildString { + append("curl -s ") + if (config.githubAuthToken != null) + append("-H \"Authorization: token ${config.githubAuthToken}\" ") + if (config.overrideGithubRelease == GithubReleaseOverride.LATEST) + append("https://api.github.com/repos/${artifact.repo}/releases | jq 'map(select(.draft == false)) | sort_by(.published_at) | last'") + else + append("https://api.github.com/repos/${artifact.repo}/releases/$releaseURL") + append(" | grep 'browser_download_url'") + }, + targetDir / "response-${artifact.repo.replace("/", "-")}-$version", + expiration = config.cacheExpirationTime.takeIf { version == "latest" } + ).getFromCacheOrEval() + + val downloadURLs = commandResult + .result + .split("\n") + .map { it.trim().removePrefix("\"browser_download_url\": \"").trim('"') } + .filter { it.contains(artifact.calculatedRegex) } + + val fullName = TextColors.yellow("github:${artifact.repo}:$version:${artifact.regex}") + if (!commandResult.wasCached) { + t.println(gray("${MSG.github} Got artifact URLs for $fullName")) + } + + return coroutineScope { + downloadURLs.map { url -> + async { HttpDownload(client, Source(artifact.source.keyInConfig, url), targetDir).download() } + }.awaitAll().flatten() + } + } +} diff --git a/src/main/kotlin/helpers/Downloads.kt b/src/main/kotlin/helpers/Downloads.kt deleted file mode 100644 index 99cee44..0000000 --- a/src/main/kotlin/helpers/Downloads.kt +++ /dev/null @@ -1,26 +0,0 @@ -package helpers - -import GithubDownload -import Keepup -import commands.DownloadedItem -import commands.Wget -import java.nio.file.Path - -/** - * Downloads a file from a [source] definition into a [targetDir] - * - * @param source Takes form of an https url, rclone remote, or `.` to ignore - */ -context(Keepup) -fun download(source: String, targetDir: Path): List { - return runCatching { - return when { - source == "." -> emptyList() - source.startsWith("github:") -> GithubDownload.from(source).download(targetDir, overrideGithubRelease) - source.matches("^https?://.*".toRegex()) -> listOfNotNull(Wget(source, targetDir)) - else -> listOfNotNull(commands.Rclone.sync(source, targetDir)) - } - } - .onFailure { System.err.println("Error downloading $source\n${it.message?.prependIndent(" ")}") } - .getOrElse { emptyList() } -} diff --git a/src/main/kotlin/helpers/DownloadsContext.kt b/src/main/kotlin/helpers/DownloadsContext.kt deleted file mode 100644 index 73648d0..0000000 --- a/src/main/kotlin/helpers/DownloadsContext.kt +++ /dev/null @@ -1,8 +0,0 @@ -package helpers - -import java.nio.file.Path - -class DownloadsContext( - val targetDir: Path, - val forceLatest: Boolean, -) diff --git a/src/main/kotlin/helpers/Helpers.kt b/src/main/kotlin/helpers/Helpers.kt index 3dc0870..13380f9 100644 --- a/src/main/kotlin/helpers/Helpers.kt +++ b/src/main/kotlin/helpers/Helpers.kt @@ -2,7 +2,7 @@ package helpers import com.typesafe.config.ConfigFactory import com.typesafe.config.ConfigRenderOptions -import commands.DownloadedItem +import downloading.DownloadResult import java.io.ByteArrayInputStream import java.io.InputStream import java.nio.file.Path @@ -36,14 +36,7 @@ fun getLeafStrings(map: Map, acc: MutableMap = mut } /** Creates a symlink for a downloaded [item] to a [dest] folder */ -fun linkToDest(dest: Path, isolatedPath: Path, item: DownloadedItem) { - (dest / item.name).createSymbolicLinkPointingTo( - (isolatedPath / item.relativePath).relativeTo(dest.absolute()) - ) +fun linkToDest(dest: Path, source: DownloadResult.HasFiles) { + val file = source.file + (dest / file.name).createSymbolicLinkPointingTo((file).relativeTo(dest.absolute())) } - -/** Removes everything between the first digit and ext of the file */ -fun String.removeVersion() = "${takeWhile { !it.isDigit() }}.${takeLastWhile { it != '.' }}" - -/** Checks if two strings are similar with their versions removed */ -fun similar(a: String, b: String): Boolean = a.removeVersion() == b.removeVersion() diff --git a/src/main/kotlin/helpers/Messages.kt b/src/main/kotlin/helpers/Messages.kt new file mode 100644 index 0000000..798a3ab --- /dev/null +++ b/src/main/kotlin/helpers/Messages.kt @@ -0,0 +1,12 @@ +package helpers + +import com.github.ajalt.mordant.rendering.TextColors.* + +object MSG { + val download = brightBlue("[Downloaded]") + val failure = brightRed("[Failure] ") + val cached = brightGreen("[Use Cached]") + val github = gray("[Github] ") + val rclone = brightBlue("[Rclone] ") + val skipped = brightYellow("[Ignoring] ") +} diff --git a/src/main/kotlin/helpers/PrintToConsole.kt b/src/main/kotlin/helpers/PrintToConsole.kt new file mode 100644 index 0000000..0d96a54 --- /dev/null +++ b/src/main/kotlin/helpers/PrintToConsole.kt @@ -0,0 +1,29 @@ +package helpers + +import com.github.ajalt.mordant.rendering.TextColors.* +import downloading.DownloadResult +import t +import kotlin.io.path.name + + +fun DownloadResult.printToConsole() { + val formattedKey = yellow(keyInConfig) + + when (this) { + is DownloadResult.Failure -> { + t.println(brightRed("${MSG.failure} for $formattedKey: $message"), stderr = true) + } + + is DownloadResult.Downloaded -> { + t.println("${overrideInfoMsg ?: MSG.download} $formattedKey ${gray("as " + file.name)}") + } + + is DownloadResult.SkippedBecauseCached -> { + t.println("${MSG.cached} $formattedKey ${gray("(${file.name})")}") + } + + is DownloadResult.SkippedBecauseSimilar -> { + t.println("${MSG.skipped} $formattedKey ${gray("similar file found: $similarTo")}") + } + } +}