-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Rewrite to use ktor and coroutines, cleanup
- Big performance improvements, both when downloading and caching values - Cache more smartly based on last edited date and content size - Lots of internal cleanup - Use shadowJar's minimized dist - Pretty up that console and add a progress bar!
- Loading branch information
Showing
25 changed files
with
458 additions
and
196 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,2 @@ | ||
| kotlin.code.style=official | ||
| version=1.2.3 | ||
| version=2.0.0-beta.1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| distributionBase=GRADLE_USER_HOME | ||
| distributionPath=wrapper/dists | ||
| distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip | ||
| distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip | ||
| zipStoreBase=GRADLE_USER_HOME | ||
| zipStorePath=wrapper/dists |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| import downloading.DownloadResult | ||
| import java.nio.file.Path | ||
| import kotlin.io.path.isRegularFile | ||
| import kotlin.io.path.listDirectoryEntries | ||
| import kotlin.io.path.name | ||
|
|
||
| class SimilarFileChecker( | ||
| val dest: Path | ||
| ) { | ||
| val names = dest | ||
| .listDirectoryEntries() | ||
| .filter { it.isRegularFile() } | ||
| .map { it.name } | ||
|
|
||
| fun findSimilarFileTo(key: String): String? { | ||
| return names.firstOrNull { similar(it, key) } | ||
| } | ||
|
|
||
| fun filterSimilarFiles(results: List<DownloadResult>): List<DownloadResult> { | ||
| return results.map { | ||
| if (it is DownloadResult.HasFiles) { | ||
| val similarFile = this.findSimilarFileTo(it.file.name) | ||
| if (similarFile != null) | ||
| return@map DownloadResult.SkippedBecauseSimilar(it.keyInConfig, similarFile) | ||
| } | ||
| it | ||
| } | ||
| } | ||
|
|
||
| /** Removes everything between the first digit and ext of the file */ | ||
| fun String.removeVersion() = "${takeWhile { !it.isDigit() }}.${takeLastWhile { it != '.' }}" | ||
|
|
||
| /** Checks if two strings are similar with their versions removed */ | ||
| fun similar(a: String, b: String): Boolean = a.removeVersion() == b.removeVersion() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,32 +1,40 @@ | ||
| package commands | ||
|
|
||
| import evalBash | ||
| import java.nio.file.LinkOption | ||
| import java.nio.file.Path | ||
| import java.time.LocalDateTime | ||
| import kotlin.io.path.* | ||
| import kotlin.time.Duration | ||
| import kotlin.time.toJavaDuration | ||
|
|
||
| class CachedCommand(val command: String, val path: Path, val expiration: Duration? = null) { | ||
| fun getFromCacheOrEval(): String { | ||
| class Result( | ||
| val wasCached: Boolean, | ||
| val result: String, | ||
| ) | ||
|
|
||
| fun getFromCacheOrEval(): Result { | ||
| val expirationPath = path.parent / (path.name + ".expiration") | ||
| // get current time | ||
| val time = LocalDateTime.now() | ||
| val expiryDate = expirationPath.takeIf { it.exists() }?.readText()?.let { LocalDateTime.parse(it) } | ||
|
|
||
| if (path.exists() && (expiryDate == null || time < expiryDate)) { | ||
| return path.readText() | ||
| return Result(true, path.readText()) | ||
| } | ||
|
|
||
| val evaluated = command.evalBash(env = mapOf()).getOrThrow() | ||
| val evaluated = command.evalBash(env = mapOf()) | ||
| .onFailure { | ||
| throw IllegalStateException("Failed to evaluate command $command, result:\n${this.stderr.joinToString("\n")}") | ||
| } | ||
| .getOrThrow() | ||
| path.deleteIfExists() | ||
| path.createParentDirectories().createFile().writeText(evaluated) | ||
| // write expiration date | ||
| if (expiration != null) { | ||
| expirationPath.deleteIfExists() | ||
| expirationPath.createFile().writeText((time + expiration.toJavaDuration()).toString()) | ||
| } | ||
| return evaluated | ||
| return Result(false, evaluated) | ||
| } | ||
| } |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,33 +1,17 @@ | ||
| package commands | ||
|
|
||
| import com.lordcodes.turtle.ShellRunException | ||
| import com.lordcodes.turtle.shellRun | ||
| import kotlinx.serialization.json.Json | ||
| import java.nio.file.Path | ||
| import kotlin.io.path.div | ||
|
|
||
| object Rclone { | ||
| val rclone = "rclone" | ||
| private val json = Json { ignoreUnknownKeys = true } | ||
|
|
||
| fun sync(source: String, targetDir: Path): DownloadedItem? { | ||
| return runCatching { | ||
| shellRun(rclone, listOf("sync", source, targetDir.toString())) | ||
| } | ||
| .map { | ||
| val name = source.substringAfterLast("/") | ||
| DownloadedItem("$targetDir/$name", name) | ||
| } | ||
| .onSuccess { println("Downloaded $source") } | ||
| .onFailure { | ||
| if (it is ShellRunException) System.err.println( | ||
| "Error downloading $source\n${ | ||
| it.message?.prependIndent( | ||
| " " | ||
| ) | ||
| }" | ||
| ) | ||
| } | ||
| .getOrNull() | ||
| fun sync(source: String, targetDir: Path): Path { | ||
| shellRun(rclone, listOf("sync", source, targetDir.toString())) | ||
| // TODO support multiple item downloads via rclone | ||
| val name = source.substringAfterLast("/") | ||
| return targetDir / name | ||
| } | ||
| } | ||
|
|
Oops, something went wrong.