diff --git a/build.gradle b/build.gradle index c62c6fde7..d93d9dbd7 100644 --- a/build.gradle +++ b/build.gradle @@ -44,7 +44,6 @@ allprojects { project.getProperty('installPath') : Paths.get(System.properties['user.home'].toString(), ".ipython", "kernels", "kotlin").toAbsolutePath().toString() ext.debugPort = 1044 - ext.configFile = "config.json" ext.debuggerConfig = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$debugPort" } @@ -69,6 +68,7 @@ dependencies { compile "org.apache.maven:maven-core:3.0.3" compile 'org.slf4j:slf4j-api:1.7.25' + compile "khttp:khttp:1.0.0" compile 'org.zeromq:jeromq:0.3.5' compile 'com.beust:klaxon:5.2' runtime 'org.slf4j:slf4j-simple:1.7.25' @@ -117,7 +117,7 @@ void createTaskForSpecs(Boolean debug) { } .join(File.pathSeparator) spec = substitute(spec, "RUNTIME_CLASSPATH", libsCp) spec = substitute(spec, "DEBUGGER_CONFIG", debug ? "\"$debuggerConfig\"," : "") - spec = substitute(spec, "LIBRARIES_PATH", "$installPath$sep$configFile") + spec = substitute(spec, "KERNEL_HOME", "$installPath") File installDir = new File("$installPath") if (!installDir.exists()) { installDir.mkdirs(); @@ -138,9 +138,9 @@ static String substitute(String spec, String template, String val) { return spec.replace("\${$template}", val.replace("\\", "\\\\")) } -task copyLibrariesConfig(type: Copy, dependsOn: cleanInstallDir) { - from configFile - into installPath +task copyLibraries(type: Copy, dependsOn: cleanInstallDir) { + from "libraries" + into Paths.get(installPath, "libraries").toString() } createTaskForSpecs(true) @@ -151,8 +151,8 @@ task installLibs(type: Copy, dependsOn: cleanInstallDir) { from configurations.deploy } -task install(dependsOn: [installKernel, installLibs, createSpecs, copyLibrariesConfig]) { +task install(dependsOn: [installKernel, installLibs, createSpecs, copyLibraries]) { } -task installDebug(dependsOn: [installKernel, installLibs, createDebugSpecs, copyLibrariesConfig]) { +task installDebug(dependsOn: [installKernel, installLibs, createDebugSpecs, copyLibraries]) { } diff --git a/config.json b/config.json deleted file mode 100644 index 46c31cee6..000000000 --- a/config.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "repositories": [ - "https://jcenter.bintray.com/", - "https://repo.maven.apache.org/maven2/", - "https://jitpack.io" - ], - "libraries": [ - { - "name": "klaxon(v=5.2)", - "dependencies": [ - "com.beust:klaxon:$v" - ], - "imports": [ - "com.beust.klaxon.*" - ], - "link": "https://github.com/cbeust/klaxon" - }, - { - "name": "lets-plot", - "link": "https://github.com/JetBrains/lets-plot-kotlin", - "repositories": [ - "https://jetbrains.bintray.com/lets-plot-maven" - ], - "dependencies": [ - "org.jetbrains.lets-plot:lets-plot-common:1.0.1-SNAPSHOT", - "org.jetbrains.lets-plot:lets-plot-kotlin-api:0.0.8-SNAPSHOT", - "org.jetbrains.lets-plot:kotlin-frontend-api:0.0.8-SNAPSHOT", - "org.jetbrains.lets-plot:lets-plot-jfx:1.0.1-SNAPSHOT" - ], - "imports": [ - "jetbrains.letsPlot.*", - "jetbrains.letsPlot.geom.*", - "jetbrains.letsPlot.stat.*" - ], - "init": [ - "fun jetbrains.letsPlot.intern.Plot.getHtml() = jetbrains.letsPlot.intern.frontendContext.FrontendContextUtil.getHtml(this)", - "DISPLAY(HTML(jetbrains.datalore.jupyter.configureScript()))" - ], - "renderers": [ - { - "class": "jetbrains.letsPlot.intern.Plot", - "result": "HTML(($it as jetbrains.letsPlot.intern.Plot).getHtml())" - } - ] - }, - { - "name": "krangl(v=-SNAPSHOT)", - "dependencies": [ - "com.github.holgerbrandl:krangl:$v" - ], - "imports": [ - "krangl.*" - ], - "init": [ - "fun krangl.DataFrame.toHTML(limit: Int = 20, truncate: Int = 50) : String\n{val sb = StringBuilder()\nsb.append(\"\")\nsb.append(\"\")\ncols.forEach { sb.append(\"\") }\nsb.append(\"\")\nrows.take(limit).forEach {\n sb.append(\"\")\n it.values.map{it.toString()}.forEach { \n val truncated = if (truncate > 0 && it.length > truncate) {\n if (truncate < 4) it.substring(0, truncate)\n else it.substring(0, truncate - 3) + \"...\"\n } else {\n it\n }\n sb.append(\"\"\"\"\"\") \n }\n sb.append(\"\")\n}\nsb.append(\"
${it.name}
$truncated
\")\nif(limit < rows.count())\n sb.append(\"

... only showing top $limit rows

\")\nsb.append(\"\")\nreturn sb.toString()}" - ], - "renderers": [ - { - "class": "krangl.SimpleDataFrame", - "result": "HTML($it.toHTML())" - } - ], - "link": "https://github.com/holgerbrandl/krangl" - }, - { - "name": "kotlin-statistics(v=-SNAPSHOT)", - "dependencies": [ - "com.github.thomasnield:kotlin-statistics:$v" - ], - "imports": [ - "org.nield.kotlinstatistics.*" - ], - "link": "https://github.com/thomasnield/kotlin-statistics" - }, - { - "name": "kravis(v=-SNAPSHOT)", - "dependencies": [ - "com.github.holgerbrandl:kravis:$v" - ], - "imports": [ - "kravis.*" - ], - "renderers": [ - { - "class": "kravis.GGPlot", - "result": "$it.show()" - } - ], - "link": "https://github.com/holgerbrandl/kravis" - }, - { - "name": "spark(scala=2.11.12,spark=2.4.4)", - "dependencies": [ - "org.apache.spark:spark-mllib_2.11:$spark", - "org.apache.spark:spark-sql_2.11:$spark", - "org.apache.spark:spark-repl_2.11:$spark", - "org.apache.spark:spark-streaming-flume-assembly_2.11:$spark", - "org.apache.spark:spark-graphx_2.11:$spark", - "org.apache.spark:spark-launcher_2.11:$spark", - "org.apache.spark:spark-catalyst_2.11:$spark", - "org.apache.spark:spark-streaming_2.11:$spark", - "org.apache.spark:spark-core_2.11:$spark", - "org.scala-lang:scala-library:$scala", - "org.scala-lang:scala-reflect:$scala", - "org.scala-lang:scala-compiler:$scala", - "org.scala-lang.modules:scala-xml_2.11:1.2.0", - "commons-io:commons-io:2.5" - ], - "imports": [ - "org.apache.spark.sql.*", - "org.apache.spark.api.java.*", - "org.apache.spark.ml.feature.*", - "org.apache.spark.sql.functions.*" - ], - "init": [ - "org.apache.log4j.Logger.getLogger(\"org\").setLevel(org.apache.log4j.Level.OFF)", - "org.apache.log4j.Logger.getLogger(\"akka\").setLevel(org.apache.log4j.Level.OFF)", - "val spark = SparkSession\n .builder()\n .appName(\"Spark example\")\n .master(\"local\")\n .getOrCreate()", - "val sc = spark.sparkContext()", - "%dumpClassesForSpark", - "fun Dataset.toHTML(limit: Int = 20, truncate: Int = 50): String {\n val sb = StringBuilder()\n\n sb.append(\"\")\n sb.append(\"\"\"\"\"\")\n sb.append(schema().fieldNames().map { \"\"}.joinToString(\"\"))\n sb.append(\"\")\n\n limit(limit).collectAsList().forEach { row ->\n sb.append(\"\")\n (0 until row.size()).map {\n row[it].toString()\n }.forEach {\n val truncated = if (truncate > 0 && it.length > truncate) {\n if (truncate < 4) it.substring(0, truncate)\n else it.substring(0, truncate - 3) + \"...\"\n } else {\n it\n }\n sb.append(\"\"\"\"\"\")\n }\n sb.append(\"\")\n }\n sb.append(\"
${it}
$truncated
\")\n if(limit < count())\n sb.append(\"

... only showing top $limit rows

\")\n sb.append(\"\")\n return sb.toString()\n}" - ], - "initCell": [ - "scala.Console.setOut(System.out)", - "scala.Console.setErr(System.err)" - ], - "renderers": [ - { - "class": "org.apache.spark.sql.Dataset", - "result": "HTML($it.toHTML())" - } - ] - }, - { - "name": "gral(v=0.11)", - "link": "https://github.com/eseifert/gral", - "dependencies": [ - "de.erichseifert.gral:gral-core:$v" - ], - "imports": [ - "de.erichseifert.gral.data.*", - "de.erichseifert.gral.data.filters.*", - "de.erichseifert.gral.graphics.*", - "de.erichseifert.gral.plots.*", - "de.erichseifert.gral.plots.lines.*", - "de.erichseifert.gral.plots.points.*", - "de.erichseifert.gral.util.*" - ], - "init": [ - "fun T.show(sizeX: Double, sizeY: Double): Any {\n val writer = de.erichseifert.gral.io.plots.DrawableWriterFactory.getInstance().get(\"image/svg+xml\")\n\n val buf = java.io.ByteArrayOutputStream()\n\n writer.write(this, buf, sizeX, sizeY)\n\n return MIME(writer.mimeType to buf.toString())\n}" - ] - }, - { - "name": "kmath(v=0.1.3)", - "link": "https://github.com/mipt-npm/kmath", - "repositories": [ - "https://dl.bintray.com/mipt-npm/scientifik" - ], - "dependencies": [ - "scientifik:kmath-core-jvm:$v" - ], - "imports": [ - "scientifik.kmath.linear.*", - "scientifik.kmath.operations.*", - "scientifik.kmath.structures.*" - ] - }, - { - "name": "koma(v=0.12)", - "link": "https://koma.kyonifer.com/index.html", - "repositories": [ - "https://dl.bintray.com/kyonifer/maven" - ], - "dependencies": [ - "com.kyonifer:koma-core-ejml:$v", - "com.kyonifer:koma-plotting:$v" - ], - "imports": [ - "koma.*", - "koma.extensions.*" - ] - } - ] -} diff --git a/kernelspec/kernel.json.template b/kernelspec/kernel.json.template index 8660d2f16..fdf4a8922 100644 --- a/kernelspec/kernel.json.template +++ b/kernelspec/kernel.json.template @@ -1,5 +1,5 @@ { - "argv": [ "java", "-jar", ${DEBUGGER_CONFIG} "${KERNEL_JAR_PATH}", "{connection_file}", "-cp=${RUNTIME_CLASSPATH}", "-libs=${LIBRARIES_PATH}"], + "argv": [ "java", "-jar", ${DEBUGGER_CONFIG} "${KERNEL_JAR_PATH}", "{connection_file}", "-cp=${RUNTIME_CLASSPATH}", "-home=${KERNEL_HOME}"], "display_name": "Kotlin", "language": "kotlin" } \ No newline at end of file diff --git a/libraries/.properties b/libraries/.properties new file mode 100644 index 000000000..045e6eae5 --- /dev/null +++ b/libraries/.properties @@ -0,0 +1 @@ +formatVersion=1 \ No newline at end of file diff --git a/libraries/gral.json b/libraries/gral.json new file mode 100644 index 000000000..83eeaf2d8 --- /dev/null +++ b/libraries/gral.json @@ -0,0 +1,21 @@ +{ + "properties": { + "v": "0.11" + }, + "link": "https://github.com/eseifert/gral", + "dependencies": [ + "de.erichseifert.gral:gral-core:$v" + ], + "imports": [ + "de.erichseifert.gral.data.*", + "de.erichseifert.gral.data.filters.*", + "de.erichseifert.gral.graphics.*", + "de.erichseifert.gral.plots.*", + "de.erichseifert.gral.plots.lines.*", + "de.erichseifert.gral.plots.points.*", + "de.erichseifert.gral.util.*" + ], + "init": [ + "fun T.show(sizeX: Double, sizeY: Double): Any {\n val writer = de.erichseifert.gral.io.plots.DrawableWriterFactory.getInstance().get(\"image/svg+xml\")\n\n val buf = java.io.ByteArrayOutputStream()\n\n writer.write(this, buf, sizeX, sizeY)\n\n return MIME(writer.mimeType to buf.toString())\n}" + ] +} diff --git a/libraries/klaxon.json b/libraries/klaxon.json new file mode 100644 index 000000000..f95a3d460 --- /dev/null +++ b/libraries/klaxon.json @@ -0,0 +1,12 @@ +{ + "properties": { + "v": "5.2" + }, + "link": "https://github.com/cbeust/klaxon", + "dependencies": [ + "com.beust:klaxon:$v" + ], + "imports": [ + "com.beust.klaxon.*" + ] +} diff --git a/libraries/kmath.json b/libraries/kmath.json new file mode 100644 index 000000000..c3e9fe295 --- /dev/null +++ b/libraries/kmath.json @@ -0,0 +1,17 @@ +{ + "properties": { + "v": "0.1.3" + }, + "link": "https://github.com/mipt-npm/kmath", + "repositories": [ + "https://dl.bintray.com/mipt-npm/scientifik" + ], + "dependencies": [ + "scientifik:kmath-core-jvm:$v" + ], + "imports": [ + "scientifik.kmath.linear.*", + "scientifik.kmath.operations.*", + "scientifik.kmath.structures.*" + ] +} diff --git a/libraries/koma.json b/libraries/koma.json new file mode 100644 index 000000000..86ffbacc6 --- /dev/null +++ b/libraries/koma.json @@ -0,0 +1,17 @@ +{ + "properties": { + "v": "0.13" + }, + "link": "https://koma.kyonifer.com/index.html", + "repositories": [ + "https://dl.bintray.com/kyonifer/maven" + ], + "dependencies": [ + "com.kyonifer:koma-core-ejml:$v", + "com.kyonifer:koma-plotting:$v" + ], + "imports": [ + "koma.*", + "koma.extensions.*" + ] +} diff --git a/libraries/kotlin-statistics.json b/libraries/kotlin-statistics.json new file mode 100644 index 000000000..b2b34706a --- /dev/null +++ b/libraries/kotlin-statistics.json @@ -0,0 +1,12 @@ +{ + "properties": { + "v": "-SNAPSHOT" + }, + "link": "https://github.com/thomasnield/kotlin-statistics", + "dependencies": [ + "com.github.thomasnield:kotlin-statistics:$v" + ], + "imports": [ + "org.nield.kotlinstatistics.*" + ] +} diff --git a/libraries/krangl.json b/libraries/krangl.json new file mode 100644 index 000000000..0d0d77b08 --- /dev/null +++ b/libraries/krangl.json @@ -0,0 +1,21 @@ +{ + "properties": { + "v": "-SNAPSHOT" + }, + "link": "https://github.com/holgerbrandl/krangl", + "dependencies": [ + "com.github.holgerbrandl:krangl:$v" + ], + "imports": [ + "krangl.*" + ], + "init": [ + "fun krangl.DataFrame.toHTML(limit: Int = 20, truncate: Int = 50) : String\n{val sb = StringBuilder()\nsb.append(\"\")\nsb.append(\"\")\ncols.forEach { sb.append(\"\") }\nsb.append(\"\")\nrows.take(limit).forEach {\n sb.append(\"\")\n it.values.map{it.toString()}.forEach { \n val truncated = if (truncate > 0 && it.length > truncate) {\n if (truncate < 4) it.substring(0, truncate)\n else it.substring(0, truncate - 3) + \"...\"\n } else {\n it\n }\n sb.append(\"\"\"\"\"\") \n }\n sb.append(\"\")\n}\nsb.append(\"
${it.name}
$truncated
\")\nif(limit < rows.count())\n sb.append(\"

... only showing top $limit rows

\")\nsb.append(\"\")\nreturn sb.toString()}" + ], + "renderers": [ + { + "class": "krangl.SimpleDataFrame", + "result": "HTML($it.toHTML())" + } + ] +} diff --git a/libraries/kravis.json b/libraries/kravis.json new file mode 100644 index 000000000..c13301f4b --- /dev/null +++ b/libraries/kravis.json @@ -0,0 +1,18 @@ +{ + "properties": { + "v": "-SNAPSHOT" + }, + "link": "https://github.com/holgerbrandl/kravis", + "dependencies": [ + "com.github.holgerbrandl:kravis:$v" + ], + "imports": [ + "kravis.*" + ], + "renderers": [ + { + "class": "kravis.GGPlot", + "result": "$it.show()" + } + ] +} diff --git a/libraries/lets-plot.json b/libraries/lets-plot.json new file mode 100644 index 000000000..a5ae93ee2 --- /dev/null +++ b/libraries/lets-plot.json @@ -0,0 +1,31 @@ +{ + "properties": { + "core": "1.0.1-SNAPSHOT", + "kotlin": "0.0.8-SNAPSHOT" + }, + "link": "https://github.com/JetBrains/lets-plot-kotlin", + "repositories": [ + "https://jetbrains.bintray.com/lets-plot-maven" + ], + "dependencies": [ + "org.jetbrains.lets-plot:lets-plot-common:$core", + "org.jetbrains.lets-plot:lets-plot-kotlin-api:$kotlin", + "org.jetbrains.lets-plot:kotlin-frontend-api:$kotlin", + "org.jetbrains.lets-plot:lets-plot-jfx:$core" + ], + "imports": [ + "jetbrains.letsPlot.*", + "jetbrains.letsPlot.geom.*", + "jetbrains.letsPlot.stat.*" + ], + "init": [ + "fun jetbrains.letsPlot.intern.Plot.getHtml() = jetbrains.letsPlot.intern.frontendContext.FrontendContextUtil.getHtml(this)", + "DISPLAY(HTML(jetbrains.datalore.jupyter.configureScript()))" + ], + "renderers": [ + { + "class": "jetbrains.letsPlot.intern.Plot", + "result": "HTML(($it as jetbrains.letsPlot.intern.Plot).getHtml())" + } + ] +} diff --git a/libraries/spark.json b/libraries/spark.json new file mode 100644 index 000000000..488a42f27 --- /dev/null +++ b/libraries/spark.json @@ -0,0 +1,46 @@ +{ + "properties": { + "scala": "2.11.12", + "spark": "2.4.4" + }, + "dependencies": [ + "org.apache.spark:spark-mllib_2.11:$spark", + "org.apache.spark:spark-sql_2.11:$spark", + "org.apache.spark:spark-repl_2.11:$spark", + "org.apache.spark:spark-streaming-flume-assembly_2.11:$spark", + "org.apache.spark:spark-graphx_2.11:$spark", + "org.apache.spark:spark-launcher_2.11:$spark", + "org.apache.spark:spark-catalyst_2.11:$spark", + "org.apache.spark:spark-streaming_2.11:$spark", + "org.apache.spark:spark-core_2.11:$spark", + "org.scala-lang:scala-library:$scala", + "org.scala-lang:scala-reflect:$scala", + "org.scala-lang:scala-compiler:$scala", + "org.scala-lang.modules:scala-xml_2.11:1.2.0", + "commons-io:commons-io:2.5" + ], + "imports": [ + "org.apache.spark.sql.*", + "org.apache.spark.api.java.*", + "org.apache.spark.ml.feature.*", + "org.apache.spark.sql.functions.*" + ], + "init": [ + "org.apache.log4j.Logger.getLogger(\"org\").setLevel(org.apache.log4j.Level.OFF)", + "org.apache.log4j.Logger.getLogger(\"akka\").setLevel(org.apache.log4j.Level.OFF)", + "val spark = SparkSession\n .builder()\n .appName(\"Spark example\")\n .master(\"local\")\n .getOrCreate()", + "val sc = spark.sparkContext()", + "%dumpClassesForSpark", + "fun Dataset.toHTML(limit: Int = 20, truncate: Int = 50): String {\n val sb = StringBuilder()\n\n sb.append(\"\")\n sb.append(\"\"\"\"\"\")\n sb.append(schema().fieldNames().map { \"\"}.joinToString(\"\"))\n sb.append(\"\")\n\n limit(limit).collectAsList().forEach { row ->\n sb.append(\"\")\n (0 until row.size()).map {\n row[it].toString()\n }.forEach {\n val truncated = if (truncate > 0 && it.length > truncate) {\n if (truncate < 4) it.substring(0, truncate)\n else it.substring(0, truncate - 3) + \"...\"\n } else {\n it\n }\n sb.append(\"\"\"\"\"\")\n }\n sb.append(\"\")\n }\n sb.append(\"
${it}
$truncated
\")\n if(limit < count())\n sb.append(\"

... only showing top $limit rows

\")\n sb.append(\"\")\n return sb.toString()\n}" + ], + "initCell": [ + "scala.Console.setOut(System.out)", + "scala.Console.setErr(System.err)" + ], + "renderers": [ + { + "class": "org.apache.spark.sql.Dataset", + "result": "HTML($it.toHTML())" + } + ] +} diff --git a/readme.md b/readme.md index 2e991374b..1980acfd2 100644 --- a/readme.md +++ b/readme.md @@ -88,7 +88,7 @@ List of supported libraries: - [koma](https://koma.kyonifer.com/index.html) - Scientific computing library - [kmath](https://github.com/mipt-npm/kmath) - Kotlin mathematical library analogous to NumPy -*The list of all supported libraries can be found in [config file](config.json)* +*The list of all supported libraries can be found in ['libraries' directory](libraries)* A definition of supported library may have a list of optional arguments that can be overriden when library is included. The major use case for library arguments is to specify particular version of library. Most library definitions default to `-SNAPSHOT` version that may be overriden in `%use` magic. @@ -127,14 +127,14 @@ Press `TAB` to get the list of suggested items for completion. 2. Run `jupyter-notebook` 3. Attach remote debugger to JVM with specified port -## Contributing +## Adding new libraries -### Support new libraries +To support new `JVM` library and make it available via `%use` magic command you need to create a library descriptor for it. -You are welcome to add support for new `Kotlin` libraries by contributing to [config.json](config.json) file. +Check ['libraries'](libraries) directory to see examples of library descriptors. -Library descriptor has the following fields: -- `name`: short name of the library with optional arguments. All library arguments must have default value specified. Syntax: `(=, =)` +Library descriptor is a `.json` file with the following fields: +- `properties`: a dictionary of properties that are used within library descriptor - `link`: a link to library homepage. This link will be displayed in `:help` command - `repositories`: a list of maven or ivy repositories to search for dependencies - `dependencies`: a list of library dependencies @@ -143,8 +143,26 @@ Library descriptor has the following fields: - `initCell`: a list of code snippets to be executed before execution of any cell - `renderers`: a list of type converters for special rendering of particular types +*All fields are optional + Fields for type renderer: - `class`: fully-qualified class name for the type to be rendered -- `result`: expression to produce output value. Source object is referenced as `$it` +- `result`: expression that produces output value. Source object is referenced as `$it` + +Name of the file is a library name that is passed to '%use' command + +Library properties can be used in any parts of library descriptor as `$property` + +To register new library descriptor: +1. For private usage - add it to local settings folder `/.jupyter_kotlin/libraries` +2. For sharing with community - commit it to ['libraries'](libraries) directory and create pull request. + +If you are maintaining some library and want to update your library descriptor, just create pull request with your update. After your request is accepted, +new version of your library will be available to all Kotlin Jupyter users immediately on next kernel startup (no kernel update is needed). + +If a library descriptor with the same name is found in several locations, the following resolution priority is used: +1. Local settings folder (highest priority) +2. ['libraries'](libraries) folder at the latest master branch of `https://github.com/Kotlin/kotlin-jupyter` repository +3. Kernel installation directory -Library arguments can be referenced in any parts of library descriptor as `$arg` \ No newline at end of file +If you don't want some library to be updated automatically, put fixed version of its library descriptor into local settings folder. \ No newline at end of file diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt index a5f7b847b..a84dc77fa 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt @@ -32,7 +32,7 @@ fun runCommand(code: String, repl: ReplForJupyter?): ResponseWithMessage { if (it.argumentsUsage != null) s += "\n Usage: %${it.name} ${it.argumentsUsage}" s } - val libraries = repl?.config?.libraries?.toList()?.joinToStringIndented { + val libraries = repl?.config?.libraries?.awaitBlocking()?.toList()?.joinToStringIndented { "${it.first} ${it.second.link ?: ""}" } ResponseWithMessage(ResponseState.Ok, textResult("Commands:\n$commands\n\nMagics\n$magics\n\nSupported libraries:\n$libraries")) diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt index 974eec1ab..e2fe9e0fa 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt @@ -1,9 +1,36 @@ package org.jetbrains.kotlin.jupyter +import com.beust.klaxon.JsonObject +import com.beust.klaxon.Parser +import khttp.responses.Response +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.async +import org.apache.commons.io.FileUtils +import org.jetbrains.kotlin.konan.parseKonanVersion +import org.json.JSONObject import org.slf4j.LoggerFactory import java.io.File +import java.nio.file.Files +import java.nio.file.Paths import kotlin.script.experimental.dependencies.RepositoryCoordinates +val LibrariesDir = "libraries" +val LocalCacheDir = "cache" +val CachedLibrariesFootprintFile = "libsCommit" + +val LocalSettingsPath = Paths.get(System.getProperty("user.home"), ".jupyter_kotlin").toString() + +val GitHubApiHost = "api.github.com" +val GitHubRepoOwner = "kotlin" +val GitHubRepoName = "kotlin-jupyter" +val GitHubBranchName = "master" +val GitHubApiPrefix = "https://$GitHubApiHost/repos/$GitHubRepoOwner/$GitHubRepoName" + +val LibraryDescriptorExt = "json" +val LibraryPropertiesFile = ".properties" +val libraryDescriptorFormatVersion = 1 + internal val log by lazy { LoggerFactory.getLogger("ikotlin") } enum class JupyterSockets { @@ -14,9 +41,21 @@ enum class JupyterSockets { iopub } +data class KernelConfig( + val ports: Array, + val transport: String, + val signatureScheme: String, + val signatureKey: String, + val pollingIntervalMillis: Long = 100, + val scriptClasspath: List = emptyList(), + val resolverConfig: ResolverConfig? +) + +val protocolVersion = "5.3" + data class TypeRenderer(val className: String, val displayCode: String?, val resultCode: String?) -data class Variable(val name: String?, val value: String?) +data class Variable(val name: String, val value: String) class LibraryDefinition(val dependencies: List, val variables: List, @@ -27,16 +66,197 @@ class LibraryDefinition(val dependencies: List, val renderers: List, val link: String?) -data class ResolverConfig(val repositories: List, val libraries: Map) +data class ResolverConfig(val repositories: List, + val libraries: Deferred>) -data class KernelConfig( - val ports: Array, - val transport: String, - val signatureScheme: String, - val signatureKey: String, - val pollingIntervalMillis: Long = 100, - val scriptClasspath: List = emptyList(), - val resolverConfig: ResolverConfig? -) +fun parseLibraryArgument(str: String): Variable { + val eq = str.indexOf('=') + return if (eq == -1) Variable("", str.trim()) + else Variable(str.substring(0, eq).trim(), str.substring(eq + 1).trim()) +} + +fun parseLibraryName(str: String): Pair> { + val brackets = str.indexOf('(') + if (brackets == -1) return str.trim() to emptyList() + val name = str.substring(0, brackets).trim() + val args = str.substring(brackets + 1, str.indexOf(')', brackets)) + .split(',') + .map(::parseLibraryArgument) + return name to args +} + +fun readLibraries(basePath: String? = null, filter: (File) -> Boolean = { true }): List> { + val parser = Parser.default() + return File(basePath, LibrariesDir) + .listFiles()?.filter { it.extension == LibraryDescriptorExt && filter(it) } + ?.map { + log.info("Loading '${it.nameWithoutExtension}' descriptor from '${it.canonicalPath}'") + it.nameWithoutExtension to parser.parse(it.canonicalPath) as JsonObject + } + .orEmpty() +} + +fun getLatestCommitToLibraries(sinceTimestamp: String?): Pair? = + log.catchAll { + var url = "$GitHubApiPrefix/commits?path=$LibrariesDir&sha=$GitHubBranchName" + if (sinceTimestamp != null) + url += "&since=$sinceTimestamp" + log.info("Checking for new commits to library descriptors at $url") + val arr = getHttp(url).jsonArray + if (arr.length() == 0) { + if (sinceTimestamp != null) + getLatestCommitToLibraries(null) + else { + log.info("Didn't find any commits to '$LibrariesDir' at $url") + null + } + } else { + val commit = arr[0] as JSONObject + val sha = commit["sha"] as String + val timestamp = ((commit["commit"] as JSONObject)["committer"] as JSONObject)["date"] as String + sha to timestamp + } + } + +fun getHttp(url: String): Response { + val response = khttp.get(url) + if (response.statusCode != 200) + throw Exception("Http request failed. Url = $url. Response = $response") + return response +} + +fun getLibraryDescriptorVersion(commitSha: String) = + log.catchAll { + val url = "$GitHubApiPrefix/contents/$LibrariesDir/$LibraryPropertiesFile?ref=$commitSha" + log.info("Checking current library descriptor format version from $url") + val response = getHttp(url) + val downloadUrl = response.jsonObject["download_url"].toString() + val downloadResult = getHttp(downloadUrl) + val result = downloadResult.text.parseIniConfig()["formatVersion"]!!.toInt() + log.info("Current library descriptor format version: $result") + result + } + +/*** + * Downloads library descriptors from GitHub to local cache if new commits in `libraries` directory were detected + */ +fun downloadNewLibraryDescriptors() { + + // Read commit hash and timestamp for locally cached libraries. + // Timestamp is used as parameter for commits request to reduce output + + val footprintFilePath = Paths.get(LocalSettingsPath, LocalCacheDir, CachedLibrariesFootprintFile).toString() + log.info("Reading commit info for which library descriptors were cached: '$footprintFilePath'") + val footprintFile = File(footprintFilePath) + val footprint = footprintFile.tryReadIniConfig() + val timestampRegex = """\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z""".toRegex() + val syncedCommitTimestamp = footprint?.get("timestamp")?.validOrNull { timestampRegex.matches(it) } + val syncedCommitSha = footprint?.get("sha") + log.info("Local libraries are cached for commit '$syncedCommitSha' at '$syncedCommitTimestamp'") + + val (latestCommitSha, latestCommitTimestamp) = getLatestCommitToLibraries(syncedCommitTimestamp) ?: return + if (latestCommitSha.equals(syncedCommitSha)) { + log.info("No new commits to library descriptors were detected") + return + } + + // Download library descriptor version + + val descriptorVersion = getLibraryDescriptorVersion(latestCommitSha) ?: return + if (descriptorVersion != libraryDescriptorFormatVersion) { + if (descriptorVersion < libraryDescriptorFormatVersion) + log.error("Incorrect library descriptor version in GitHub repository: $descriptorVersion") + else + log.warn("Kotlin Kernel needs to be updated to the latest version. Couldn't download new library descriptors from GitHub repository because their format was changed") + return + } + + // Download library descriptors + + log.info("New commits to library descriptors were detected. Downloading library descriptors for commit $latestCommitSha") + + val libraries = log.catchAll { + val url = "$GitHubApiPrefix/contents/$LibrariesDir?ref=$latestCommitSha" + log.info("Requesting the list of library descriptors at $url") + val response = getHttp(url) + val filenameRegex = """[\w.-]+\.$LibraryDescriptorExt""".toRegex() + + response.jsonArray.mapNotNull { + val o = it as JSONObject + val filename = o["name"] as String + if (filenameRegex.matches(filename)) { + val libUrl = o["download_url"].toString() + log.info("Downloading '$filename' from $libUrl") + val res = getHttp(libUrl) + val text = res.jsonObject.toString() + filename to text + } else null + } + } ?: return + + // Save library descriptors to local cache + + val librariesPath = Paths.get(LocalSettingsPath, LocalCacheDir, LibrariesDir) + val librariesDir = librariesPath.toFile() + log.info("Saving ${libraries.count()} library descriptors to local cache at '$librariesPath'") + try { + FileUtils.deleteDirectory(librariesDir) + Files.createDirectories(librariesPath) + libraries.forEach { + File(librariesDir.toString(), it.first).writeText(it.second) + } + footprintFile.writeText(""" + timestamp=$latestCommitTimestamp + sha=$latestCommitSha + """.trimIndent()) + } catch (e: Exception) { + log.error("Failed to write downloaded library descriptors to local cache:", e) + log.catchAll { FileUtils.deleteDirectory(librariesDir) } + } +} + +fun getLibrariesJsons(homeDir: String): Map { + + downloadNewLibraryDescriptors() + + val pathsToCheck = arrayOf(LocalSettingsPath, + Paths.get(LocalSettingsPath, LocalCacheDir).toString(), + homeDir) + + val librariesMap = mutableMapOf() + + pathsToCheck.forEach { + readLibraries(it) { !librariesMap.containsKey(it.nameWithoutExtension) } + .forEach { librariesMap.put(it.first, it.second) } + } + + return librariesMap +} + +fun loadResolverConfig(homeDir: String) = ResolverConfig(defaultRepositories, GlobalScope.async { + parserLibraryDescriptors(getLibrariesJsons(homeDir)) +}) + +val defaultRepositories = arrayOf( + "https://jcenter.bintray.com/", + "https://repo.maven.apache.org/maven2/", + "https://jitpack.io" +).map { RepositoryCoordinates(it) } + +fun parserLibraryDescriptors(libJsons: Map): Map { + return libJsons.mapValues { + LibraryDefinition( + dependencies = it.value.array("dependencies")?.toList().orEmpty(), + variables = it.value.obj("properties")?.map { Variable(it.key, it.value.toString()) }.orEmpty(), + imports = it.value.array("imports")?.toList().orEmpty(), + repositories = it.value.array("repositories")?.toList().orEmpty(), + init = it.value.array("init")?.toList().orEmpty(), + initCell = it.value.array("initCell")?.toList().orEmpty(), + renderers = it.value.array("renderers")?.map { + TypeRenderer(it.string("class")!!, it.string("display"), it.string("result")) + }?.toList().orEmpty(), + link = it.value.string("link") + ) + } +} -val protocolVersion = "5.3" diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt index b1d87bf8d..f342e9d13 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt @@ -5,27 +5,24 @@ import com.beust.klaxon.Parser import java.io.File import java.util.concurrent.atomic.AtomicLong import kotlin.concurrent.thread -import kotlin.script.experimental.dependencies.RepositoryCoordinates import kotlin.script.experimental.jvm.util.classpathFromClassloader -val DefaultConfigFile = "config.json" - data class KernelArgs(val cfgFile: File, val scriptClasspath: List, - val libs: File?) + val homeDir: File?) private fun parseCommandLine(vararg args: String): KernelArgs { var cfgFile: File? = null var classpath: List? = null - var libsJson: File? = null + var homeDir: File? = null args.forEach { when { it.startsWith("-cp=") || it.startsWith("-classpath=") -> { if (classpath != null) throw IllegalArgumentException("classpath already set to ${classpath!!.joinToString(File.pathSeparator)}") classpath = it.substringAfter('=').split(File.pathSeparator).map { File(it) } } - it.startsWith("-libs=") -> { - libsJson = File(it.substringAfter('=')) + it.startsWith("-home=") -> { + homeDir = File(it.substringAfter('=')) } else -> { if (cfgFile != null) throw IllegalArgumentException("config file already set to $cfgFile") @@ -35,7 +32,7 @@ private fun parseCommandLine(vararg args: String): KernelArgs { } if (cfgFile == null) throw IllegalArgumentException("config file is not provided") if (!cfgFile!!.exists() || !cfgFile!!.isFile) throw IllegalArgumentException("invalid config file $cfgFile") - return KernelArgs(cfgFile!!, classpath ?: emptyList(), libsJson) + return KernelArgs(cfgFile!!, classpath ?: emptyList(), homeDir) } fun printClassPath() { @@ -48,49 +45,12 @@ fun printClassPath() { log.info("Current classpath: " + cp.joinToString()) } -fun parseLibraryName(str: String): Pair> { - val pattern = """\w+(\w+)?""".toRegex().matches(str) - val brackets = str.indexOf('(') - if (brackets == -1) return str.trim() to emptyList() - val name = str.substring(0, brackets).trim() - val args = str.substring(brackets + 1, str.indexOf(')', brackets)) - .split(',') - .map { - val eq = it.indexOf('=') - if (eq == -1) Variable(it.trim(), null) - else Variable(it.substring(0, eq).trim(), it.substring(eq + 1).trim()) - } - return name to args -} - -fun readResolverConfig(file: File = File(DefaultConfigFile)): ResolverConfig = - parseResolverConfig(Parser().parse(file.canonicalPath) as JsonObject) - -fun parseResolverConfig(json: JsonObject): ResolverConfig { - val repos = json.array("repositories")?.map { RepositoryCoordinates(it) }.orEmpty() - val artifacts = json.array("libraries")?.map { - val (name, variables) = parseLibraryName(it.string("name")!!) - name to LibraryDefinition( - dependencies = it.array("dependencies")?.toList().orEmpty(), - variables = variables, - imports = it.array("imports")?.toList().orEmpty(), - repositories = it.array("repositories")?.toList().orEmpty(), - init = it.array("init")?.toList().orEmpty(), - initCell = it.array("initCell")?.toList().orEmpty(), - renderers = it.array("renderers")?.map { - TypeRenderer(it.string("class")!!, it.string("display"), it.string("result")) - }?.toList().orEmpty(), - link = it.string("link") - ) - }?.toMap() - return ResolverConfig(repos, artifacts.orEmpty()) -} - fun main(vararg args: String) { try { log.info("Kernel args: "+ args.joinToString { it }) - val (cfgFile, scriptClasspath, librariesConfigFile) = parseCommandLine(*args) - val cfgJson = Parser().parse(cfgFile.canonicalPath) as JsonObject + val (cfgFile, scriptClasspath, homeDir) = parseCommandLine(*args) + val rootPath = homeDir!!.toString() + val cfgJson = Parser.default().parse(cfgFile.canonicalPath) as JsonObject fun JsonObject.getInt(field: String): Int = int(field) ?: throw RuntimeException("Cannot find $field in $cfgFile") val sigScheme = cfgJson.string("signature_scheme") @@ -102,7 +62,7 @@ fun main(vararg args: String) { signatureScheme = sigScheme ?: "hmac1-sha256", signatureKey = if (sigScheme == null || key == null) "" else key, scriptClasspath = scriptClasspath, - resolverConfig = librariesConfigFile?.let { readResolverConfig(it) } + resolverConfig = loadResolverConfig(rootPath) )) } catch (e: Exception) { log.error("exception running kernel with args: \"${args.joinToString()}\"", e) diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt index ef29dce71..feb762bdd 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt @@ -20,24 +20,21 @@ class LibrariesProcessor { * @return A name-to-value map of library arguments */ private fun substituteArguments(parameters: List, arguments: List): Map { - val firstNamed = arguments.indexOfFirst { it.name != null } - if (firstNamed != -1 && arguments.asSequence().drop(firstNamed).any { it.name == null }) - throw ReplCompilerException("Mixing named and positional arguments is not allowed") - val parameterNames = parameters.map { it.name!! }.toSet() val result = mutableMapOf() - for (i in 0 until arguments.count()) { - if (i >= parameters.count()) + if (arguments.any { it.name.isEmpty() }) { + if (parameters.count() != 1) + throw ReplCompilerException("Unnamed argument is allowed only if library has a single property") + if (arguments.count() != 1) throw ReplCompilerException("Too many arguments") - val name = arguments[i].name?.also { - if (!parameterNames.contains(it)) throw ReplCompilerException("Can not find parameter with name '$it'") - } ?: parameters[i].name!! + result[parameters[0].name] = arguments[0].value + return result + } - if (result.containsKey(name)) throw ReplCompilerException("An argument for parameter '$name' is already passed") - result[name] = arguments[i].value!! + arguments.forEach { + result[it.name] = it.value } parameters.forEach { - if (!result.containsKey(it.name!!)) { - if (it.value == null) throw ReplCompilerException("No value passed for parameter '${it.name}'") + if (!result.containsKey(it.name)) { result[it.name] = it.value } } @@ -95,11 +92,10 @@ class LibrariesProcessor { splitLibraryCalls(arg).forEach { val (name, vars) = parseLibraryName(it) - val library = repl.config?.libraries?.get(name) ?: throw ReplCompilerException("Unknown library '$name'") + val library = repl.config?.libraries?.awaitBlocking()?.get(name) + ?: throw ReplCompilerException("Unknown library '$name'") - // treat single strings in parsed arguments as values, not names - val arguments = vars.map { if (it.value == null) Variable(null, it.name) else it } - val mapping = substituteArguments(library.variables, arguments) + val mapping = substituteArguments(library.variables, vars) processedLibraries.add(LibraryWithCode(library, generateCode(repl, library, mapping))) } diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt index dd76390b2..1e26b868f 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt @@ -42,7 +42,11 @@ class ReplForJupyter(val scriptClasspath: List = emptyList(), private val resolver = JupyterScriptDependenciesResolver(config) - private val renderers = config?.let { it.libraries.flatMap { it.value.renderers } }?.map { it.className to it }?.toMap().orEmpty() + private val renderers = config?.let { + it.libraries.asyncLet { + it.flatMap { it.value.renderers }.map { it.className to it }.toMap() + } + } private val includedLibraries = mutableSetOf() @@ -88,9 +92,8 @@ class ReplForJupyter(val scriptClasspath: List = emptyList(), val kt = KotlinType(receiver.javaClass.canonicalName) implicitReceivers.invoke(listOf(kt)) - val classes = listOf(/*receiver.javaClass,*/ ScriptTemplateWithDisplayHelpers::class.java) - val classPath = classes.asSequence().map { it.protectionDomain.codeSource.location.path }.joinToString(":") - compilerOptions.invoke(listOf("-classpath", classPath, "-jvm-target", "1.8")) + log.info("Classpath for compiler options: none") + compilerOptions.invoke(listOf("-jvm-target", "1.8")) } } @@ -123,6 +126,7 @@ class ReplForJupyter(val scriptClasspath: List = emptyList(), val scriptClassloader = URLClassLoader(scriptClasspath.map { it.toURI().toURL() }.toTypedArray(), filteringClassLoader) baseClassLoader(scriptClassloader) } + constructorArgs() } private var executionCounter = 0 @@ -167,7 +171,7 @@ class ReplForJupyter(val scriptClasspath: List = emptyList(), init { // TODO: to be removed after investigation of https://github.com/kotlin/kotlin-jupyter/issues/24 - eval("1") + doEval("1") } fun eval(code: String, jupyterId: Int = -1): EvalResult { @@ -226,8 +230,9 @@ class ReplForJupyter(val scriptClasspath: List = emptyList(), } } - if (result != null) { - renderers[result.javaClass.canonicalName]?.let { + if (result != null && renderers != null) { + val resultType = result.javaClass.canonicalName + renderers.awaitBlocking()[resultType]?.let { it.displayCode?.let { doEval(it.replace("\$it", "res$replId")).value?.let(displays::add) } diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/resolver.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/resolver.kt index c5fb74c77..3aaa421ff 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/resolver.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/resolver.kt @@ -4,13 +4,22 @@ import jupyter.kotlin.DependsOn import jupyter.kotlin.Repository import kotlinx.coroutines.runBlocking import org.jetbrains.kotlin.mainKts.impl.IvyResolver +import org.slf4j.LoggerFactory import java.io.File import kotlin.script.dependencies.ScriptContents -import kotlin.script.experimental.api.* -import kotlin.script.experimental.dependencies.* +import kotlin.script.experimental.api.ResultWithDiagnostics +import kotlin.script.experimental.api.ScriptDiagnostic +import kotlin.script.experimental.api.asSuccess +import kotlin.script.experimental.api.makeFailureResult +import kotlin.script.experimental.dependencies.CompoundDependenciesResolver +import kotlin.script.experimental.dependencies.ExternalDependenciesResolver +import kotlin.script.experimental.dependencies.FileSystemDependenciesResolver +import kotlin.script.experimental.dependencies.tryAddRepository open class JupyterScriptDependenciesResolver(resolverConfig: ResolverConfig?) { + private val log by lazy { LoggerFactory.getLogger("resolver") } + private val resolver: ExternalDependenciesResolver init { @@ -33,17 +42,30 @@ open class JupyterScriptDependenciesResolver(resolverConfig: ResolverConfig?) { script.annotations.forEach { annotation -> when (annotation) { is Repository -> { + log.info("Adding repository: ${annotation.value}") if (!resolver.tryAddRepository(annotation.value)) throw IllegalArgumentException("Illegal argument for Repository annotation: $annotation") } is DependsOn -> { - val result = runBlocking { resolver.resolve(annotation.value) } - when (result) { - is ResultWithDiagnostics.Failure -> scriptDiagnostics.add(ScriptDiagnostic("Failed to resolve dependencies:\n" + result.reports.joinToString("\n") { it.message })) - is ResultWithDiagnostics.Success -> { - addedClasspath.addAll(result.value) - classpath.addAll(result.value) + log.info("Resolving ${annotation.value}") + try { + val result = runBlocking { resolver.resolve(annotation.value) } + when (result) { + is ResultWithDiagnostics.Failure -> { + val diagnostics = ScriptDiagnostic("Failed to resolve ${annotation.value}:\n" + result.reports.joinToString("\n") { it.message }) + log.warn(diagnostics.message, diagnostics.exception) + scriptDiagnostics.add(diagnostics) + } + is ResultWithDiagnostics.Success -> { + log.info("Resolved: " + result.value.joinToString()) + addedClasspath.addAll(result.value) + classpath.addAll(result.value) + } } + } catch (e: Exception) { + val diagnostic = ScriptDiagnostic("Unhandled exception during resolve", exception = e) + log.error(diagnostic.message, e) + scriptDiagnostics.add(diagnostic) } } else -> throw Exception("Unknown annotation ${annotation.javaClass}") diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt new file mode 100644 index 000000000..c5550e40c --- /dev/null +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt @@ -0,0 +1,53 @@ +package org.jetbrains.kotlin.jupyter + +import com.beust.klaxon.JsonObject +import com.beust.klaxon.Parser +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.async +import kotlinx.coroutines.runBlocking +import org.json.JSONObject +import org.slf4j.Logger +import java.io.File +import java.io.StringReader +import javax.xml.bind.JAXBElement + +fun catchAll(body: () -> T): T? = try { + body() +} catch (e: Exception) { + null +} + +fun Logger.catchAll(msg: String = "", body: () -> T): T? = try { + body() +} catch (e: Exception) { + this.error(msg, e) + null +} + +fun T.validOrNull(predicate: (T) -> Boolean): T? = if (predicate(this)) this else null + +fun T.asDeferred(): Deferred = this.let { GlobalScope.async { it } } + +fun File.existsOrNull() = if (exists()) this else null + +fun Deferred.asyncLet(selector: suspend (T) -> R): Deferred = this.let { + GlobalScope.async { + selector(it.await()) + } +} + +fun Deferred.awaitBlocking(): T = if (isCompleted) getCompleted() else runBlocking { await() } + +fun String.parseIniConfig() = + split("\n").map { it.split('=') }.filter { it.count() == 2 }.map { it[0] to it[1] }.toMap() + +fun File.tryReadIniConfig() = + existsOrNull()?.let { + catchAll { it.readText().parseIniConfig() } + } + +fun readJson(path: String) = + Parser.default().parse(path) as JsonObject + +fun JSONObject.toJsonObject() = Parser.default().parse(StringReader(toString())) as JsonObject \ No newline at end of file diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseArgumentsTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseArgumentsTests.kt index 8e05ce7a8..ee7cc57bf 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseArgumentsTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseArgumentsTests.kt @@ -18,18 +18,18 @@ class ParseArgumentsTests { val (name, args) = parseLibraryName("lib(arg1)") Assert.assertEquals("lib", name) Assert.assertEquals(1, args.count()) - Assert.assertEquals("arg1", args[0].name) - Assert.assertNull(args[0].value) + Assert.assertEquals("arg1", args[0].value) + Assert.assertEquals("", args[0].name) } @Test fun test3() { - val (name, args) = parseLibraryName("lib (arg1 = 1.2, arg2)") + val (name, args) = parseLibraryName("lib (arg1 = 1.2, arg2 = val2)") Assert.assertEquals("lib", name) Assert.assertEquals(2, args.count()) Assert.assertEquals("arg1", args[0].name) Assert.assertEquals("1.2", args[0].value) Assert.assertEquals("arg2", args[1].name) - Assert.assertNull(args[1].value) + Assert.assertEquals("val2", args[1].value) } } \ No newline at end of file diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt index 47bab98fc..e08145d33 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt @@ -2,11 +2,10 @@ package org.jetbrains.kotlin.jupyter.test import com.beust.klaxon.JsonObject import com.beust.klaxon.Parser -import jupyter.kotlin.DisplayResult import jupyter.kotlin.MimeTypedResult -import org.jetbrains.kotlin.jupyter.ReplForJupyter -import org.jetbrains.kotlin.jupyter.parseResolverConfig -import org.jetbrains.kotlin.jupyter.readResolverConfig +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.async +import org.jetbrains.kotlin.jupyter.* import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResultSuccess import org.junit.Assert import org.junit.Test @@ -16,6 +15,9 @@ import kotlin.test.assertNotNull class ReplTest { + fun replWithResolver() = ReplForJupyter(classpath, ResolverConfig(defaultRepositories, + parserLibraryDescriptors(readLibraries().toMap()).asDeferred())) + @Test fun TestRepl() { val repl = ReplForJupyter(classpath) @@ -73,14 +75,14 @@ class ReplTest { @Test fun TestUseMagic() { - val config = """ - { - "libraries": [ + val lib1 = "mylib" to """ { - "name": "mylib(v1, v2=2.3)", + "properties": { + "v1": "0.2" + }, "dependencies": [ - "artifact1:""" + "\$v1" + """", - "artifact2:""" + "\$v2" + """" + "artifact1:${'$'}v1", + "artifact2:${'$'}v1" ], "imports": [ "package1", @@ -90,23 +92,27 @@ class ReplTest { "code1", "code2" ] - }, - { - "name": "other(a=temp, b=test)", - "dependencies": [ - "path-""" + "\$a" + """", - "path-""" + "\$b" + """" - ], - "imports": [ - "otherPackage" - ] - } - ] - } + }""".trimIndent() + val lib2 = "other" to """ + { + "properties": { + "a": "temp", + "b": "test" + }, + "dependencies": [ + "path-${'$'}a", + "path-${'$'}b" + ], + "imports": [ + "otherPackage" + ] + } """.trimIndent() - val json = Parser().parse(StringBuilder(config)) as JsonObject - val replConfig = parseResolverConfig(json) - val repl = ReplForJupyter(classpath, replConfig) + val parser = Parser.default() + + val libJsons = arrayOf(lib1, lib2).map { it.first to parser.parse(StringBuilder(it.second)) as JsonObject }.toMap() + + val repl = ReplForJupyter(classpath, ResolverConfig(defaultRepositories, parserLibraryDescriptors(libJsons).asDeferred())) val res = repl.preprocessCode("%use mylib(1.0), other(b=release, a=debug)").trimIndent() val libs = repl.librariesCodeGenerator.getProcessedLibraries() assertEquals("", res) @@ -114,7 +120,7 @@ class ReplTest { arrayOf( """ @file:DependsOn("artifact1:1.0") - @file:DependsOn("artifact2:2.3") + @file:DependsOn("artifact2:1.0") import package1 import package2 code1 @@ -132,7 +138,7 @@ class ReplTest { @Test fun TestLetsPlot() { - val repl = ReplForJupyter(classpath, readResolverConfig()) + val repl = replWithResolver() val code1 = "%use lets-plot" val code2 = """lets_plot(mapOf("cat" to listOf("a", "b")))""" val res1 = repl.eval(code1) @@ -149,7 +155,7 @@ class ReplTest { @Test fun TestTwoLibrariesInUse() { - val repl = ReplForJupyter(classpath, readResolverConfig()) + val repl = replWithResolver() val code = "%use lets-plot, krangl" val res = repl.eval(code) assertEquals(1, res.displayValues.count()) @@ -158,7 +164,7 @@ class ReplTest { @Test //TODO: https://github.com/Kotlin/kotlin-jupyter/issues/25 fun TestKranglImportInfixFun() { - val repl = ReplForJupyter(classpath, readResolverConfig()) + val repl = replWithResolver() val code = """%use krangl "a" to {it["a"]}""" val res = repl.eval(code)