diff --git a/docs/libraries.md b/docs/libraries.md index f10da335..d484caaa 100644 --- a/docs/libraries.md +++ b/docs/libraries.md @@ -933,13 +933,39 @@ resources. This file should contain FQNs of all integration classes in the JSON { "fqn": "org.jetbrains.kotlinx.jupyter.example.GettingStartedIntegration" } - ] + ], + "descriptors": [] } ``` Classes derived from the `LibraryDefinition` interface should be added to the `definitions` array. Classes derived from the `LibraryDefinitionProducer` interface should be added to the `producers` array. +Additionally, you're allowed to add full custom library descriptors to the `descriptors` array. +This is an advanced option that allows you to load additional dependencies or add additional imports using the +[library descriptor API](#creating-a-library-descriptor). +This can be particularly useful for libraries that cannot depend on the jupyter-api artifact but still want to modify +the kernel behavior when loaded. + +For example: + +```json +{ + "definitions": [], + "producers": [], + "descriptors": [ + { + "dependencies": [ + "my.library:my-library-jupyter-integration:1.0.0" + ], + "init": [ + "DISPLAY(\"Implicitly loading Jupyter integration for MyLibrary...\")" + ] + } + ] +} +``` + For more information, see: * [Libraries repository](https://github.com/Kotlin/kotlin-jupyter-libraries) diff --git a/jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/libraries/Scanning.kt b/jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/libraries/Scanning.kt index 9d16d238..9cbf898f 100644 --- a/jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/libraries/Scanning.kt +++ b/jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/libraries/Scanning.kt @@ -1,6 +1,7 @@ package org.jetbrains.kotlinx.jupyter.api.libraries import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonObject import org.jetbrains.kotlinx.jupyter.api.TypeName /** @@ -48,9 +49,13 @@ data class LibrariesProducerDeclaration( /** * Serialized form of this class instance is a correct content of * [KOTLIN_JUPYTER_LIBRARIES_FILE_NAME] file, and vice versa. + * + * @param descriptors List of library descriptor JSON objects adhering to + * `org.jetbrains.kotlinx.jupyter.libraries.LibraryDescriptor`. */ @Serializable data class LibrariesScanResult( val definitions: List = emptyList(), val producers: List = emptyList(), + val descriptors: List = emptyList(), ) diff --git a/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/ApiGradlePlugin.kt b/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/ApiGradlePlugin.kt index 7cd6ab25..09781e5c 100644 --- a/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/ApiGradlePlugin.kt +++ b/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/ApiGradlePlugin.kt @@ -12,7 +12,6 @@ import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension import org.jetbrains.kotlin.gradle.targets.jvm.KotlinJvmTarget import org.jetbrains.kotlinx.jupyter.api.plugin.tasks.JupyterApiResourcesTask import org.jetbrains.kotlinx.jupyter.api.plugin.util.addMavenCentralIfDoesNotExist -import org.jetbrains.kotlinx.jupyter.api.plugin.util.getBuildDirectory import org.jetbrains.kotlinx.jupyter.api.plugin.util.whenAdded class ApiGradlePlugin : Plugin { @@ -20,9 +19,6 @@ class ApiGradlePlugin : Plugin { with(target) { val pluginExtension = KotlinJupyterPluginExtension(target) extensions.add(KotlinJupyterPluginExtension.NAME, pluginExtension) - - val jupyterBuildPath = getBuildDirectory().resolve(FQNS_PATH) - jupyterBuildPath.mkdirs() pluginExtension.addDependenciesIfNeeded() repositories { @@ -63,8 +59,4 @@ class ApiGradlePlugin : Plugin { } } } - - companion object { - const val FQNS_PATH = "generated/jupyter/fqns" - } } diff --git a/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/KotlinJupyterPluginExtension.kt b/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/KotlinJupyterPluginExtension.kt index b3cfdb80..a95d1674 100644 --- a/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/KotlinJupyterPluginExtension.kt +++ b/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/KotlinJupyterPluginExtension.kt @@ -1,6 +1,7 @@ package org.jetbrains.kotlinx.jupyter.api.plugin import org.gradle.api.Project +import org.intellij.lang.annotations.Language import org.jetbrains.kotlinx.jupyter.api.plugin.util.FQNAware import org.jetbrains.kotlinx.jupyter.api.plugin.util.LibrariesScanResult import org.jetbrains.kotlinx.jupyter.api.plugin.util.configureDependency @@ -15,11 +16,13 @@ class KotlinJupyterPluginExtension( private val libraryProducers: MutableSet = mutableSetOf() private val libraryDefinitions: MutableSet = mutableSetOf() + private val libraryDescriptors: MutableSet = mutableSetOf() internal val libraryFqns get() = LibrariesScanResult( definitions = libraryDefinitions, producers = libraryProducers, + descriptors = libraryDescriptors, ) internal fun addDependenciesIfNeeded() { @@ -42,6 +45,7 @@ class KotlinJupyterPluginExtension( /** * Add adding library integrations by specifying their fully qualified names */ + @Suppress("unused") fun integrations(action: IntegrationsSpec.() -> Unit) { IntegrationsSpec().apply(action) } @@ -56,6 +60,13 @@ class KotlinJupyterPluginExtension( fun definition(className: String) { libraryDefinitions.add(FQNAware(className)) } + + @Suppress("unused") + fun descriptor( + @Language("JSON") descriptor: String, + ) { + libraryDescriptors.add(descriptor) + } } companion object { diff --git a/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/tasks/JupyterApiResourcesTask.kt b/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/tasks/JupyterApiResourcesTask.kt index 878a569d..c6fdbebb 100644 --- a/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/tasks/JupyterApiResourcesTask.kt +++ b/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/tasks/JupyterApiResourcesTask.kt @@ -5,7 +5,6 @@ import org.gradle.api.DefaultTask import org.gradle.api.tasks.Input import org.gradle.api.tasks.OutputDirectory import org.gradle.api.tasks.TaskAction -import org.jetbrains.kotlinx.jupyter.api.plugin.ApiGradlePlugin import org.jetbrains.kotlinx.jupyter.api.plugin.KotlinJupyterPluginExtension import org.jetbrains.kotlinx.jupyter.api.plugin.util.FQNAware import org.jetbrains.kotlinx.jupyter.api.plugin.util.LibrariesScanResult @@ -30,6 +29,13 @@ open class JupyterApiResourcesTask : DefaultTask() { @Input var libraryDefinitions: List = emptyList() + /** + * List of JSON library descriptors in the + * [library descriptor format](https://github.com/Kotlin/kotlin-jupyter/blob/master/docs/libraries.md#creating-a-library-descriptor). + */ + @Input + var libraryDescriptors: List = emptyList() + @OutputDirectory val outputDir: File = project.getBuildDirectory().resolve("jupyterProcessedResources") @@ -42,12 +48,10 @@ open class JupyterApiResourcesTask : DefaultTask() { LibrariesScanResult( definitions = libraryDefinitions.map { FQNAware(it) }.toSet(), producers = libraryProducers.map { FQNAware(it) }.toSet(), + descriptors = libraryDescriptors.toSet(), ) - val resultObject = - jupyterExtensionScanResult + - taskScanResult + - getScanResultFromAnnotations() + val resultObject = jupyterExtensionScanResult + taskScanResult val json = Gson().toJson(resultObject) val jupyterDir = outputDir.resolve("META-INF/kotlin-jupyter-libraries") @@ -56,28 +60,10 @@ open class JupyterApiResourcesTask : DefaultTask() { libFile.writeText(json) } - private fun getScanResultFromAnnotations(): LibrariesScanResult { - val path = project.getBuildDirectory().resolve(ApiGradlePlugin.FQNS_PATH) - - fun fqns(name: String): Set { - val file = path.resolve(name) - if (!file.exists()) return emptySet() - return file - .readLines() - .filter { it.isNotBlank() } - .map { FQNAware(it) } - .toSet() - } - - return LibrariesScanResult( - fqns("definitions"), - fqns("producers"), - ) - } - operator fun LibrariesScanResult.plus(other: LibrariesScanResult): LibrariesScanResult = LibrariesScanResult( - definitions + other.definitions, - producers + other.producers, + definitions = definitions + other.definitions, + producers = producers + other.producers, + descriptors = descriptors + other.descriptors, ) } diff --git a/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/util/LibrariesUtil.kt b/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/util/LibrariesUtil.kt index 84bcae23..7b753750 100644 --- a/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/util/LibrariesUtil.kt +++ b/jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/util/LibrariesUtil.kt @@ -7,6 +7,7 @@ data class FQNAware( class LibrariesScanResult( val definitions: Set = emptySet(), val producers: Set = emptySet(), + val descriptors: Set = emptySet(), ) val emptyScanResult = LibrariesScanResult() diff --git a/jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/LibrariesScanner.kt b/jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/LibrariesScanner.kt index fb2f3d13..fdd9b49a 100644 --- a/jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/LibrariesScanner.kt +++ b/jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/LibrariesScanner.kt @@ -1,6 +1,7 @@ package org.jetbrains.kotlinx.jupyter.libraries import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject import org.jetbrains.kotlinx.jupyter.api.KotlinKernelHost import org.jetbrains.kotlinx.jupyter.api.LibraryLoader import org.jetbrains.kotlinx.jupyter.api.Notebook @@ -13,6 +14,7 @@ import org.jetbrains.kotlinx.jupyter.api.libraries.LibrariesInstantiable import org.jetbrains.kotlinx.jupyter.api.libraries.LibrariesProducerDeclaration import org.jetbrains.kotlinx.jupyter.api.libraries.LibrariesScanResult import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinition +import org.jetbrains.kotlinx.jupyter.api.libraries.Variable import org.jetbrains.kotlinx.jupyter.config.errorForUser import org.jetbrains.kotlinx.jupyter.protocol.api.KernelLoggerFactory import org.jetbrains.kotlinx.jupyter.protocol.api.getLogger @@ -24,9 +26,11 @@ class LibrariesScanner( loggerFactory: KernelLoggerFactory, ) : LibraryLoader { private val logger = loggerFactory.getLogger(this::class) + private val jsonParser = Json { ignoreUnknownKeys = true } private val processedFQNs = mutableSetOf() private val discardedFQNs = mutableSetOf() + private val processedDescriptorHashes = mutableSetOf() private fun > Iterable.filterNamesToLoad( host: KotlinKernelHost, @@ -46,10 +50,14 @@ class LibrariesScanner( discardedFQNs.add(typeName) false } + null -> typeName !in discardedFQNs && processedFQNs.add(typeName) } } + /** Makes sure each unique descriptor is only loaded once by caching their hashes in [processedDescriptorHashes]. */ + private fun Iterable.filterDescriptorsToLoad() = filter { processedDescriptorHashes.add(it.hashCode()) } + fun addLibrariesFromClassLoader( classLoader: ClassLoader, host: KotlinKernelHost, @@ -83,17 +91,22 @@ class LibrariesScanner( integrationTypeNameRules: List> = listOf(), ): LibrariesScanResult { val results = - classLoader.getResources("$KOTLIN_JUPYTER_RESOURCES_PATH/$KOTLIN_JUPYTER_LIBRARIES_FILE_NAME").toList().map { url -> - val contents = url.readText() - Json.decodeFromString(contents) - } + classLoader + .getResources("$KOTLIN_JUPYTER_RESOURCES_PATH/$KOTLIN_JUPYTER_LIBRARIES_FILE_NAME") + .toList() + .map { url -> + val contents = url.readText() + jsonParser.decodeFromString(contents) + } val definitions = mutableListOf() val producers = mutableListOf() + val descriptors = mutableListOf() for (result in results) { definitions.addAll(result.definitions) producers.addAll(result.producers) + descriptors.addAll(result.descriptors) } fun > Iterable.filterNames() = filterNamesToLoad(host, integrationTypeNameRules) @@ -101,6 +114,7 @@ class LibrariesScanner( return LibrariesScanResult( definitions.filterNames(), producers.filterNames(), + descriptors.filterDescriptorsToLoad(), ) } @@ -140,6 +154,14 @@ class LibrariesScanner( } } } + + scanResult.descriptors.forEach { + val descriptor = parseLibraryDescriptor(it) + val arguments = libraryOptions.map { (name, value) -> Variable(name, value) } + val definition = descriptor.convertToDefinition(arguments) + definitions.add(definition) + } + return definitions } diff --git a/jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/Parsing.kt b/jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/Parsing.kt index 4253c265..74440daf 100644 --- a/jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/Parsing.kt +++ b/jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/Parsing.kt @@ -2,14 +2,31 @@ package org.jetbrains.kotlinx.jupyter.libraries import kotlinx.serialization.SerializationException import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.decodeFromJsonElement import org.jetbrains.kotlinx.jupyter.api.exceptions.ReplException import org.jetbrains.kotlinx.jupyter.protocol.api.KernelLoggerFactory -fun parseLibraryDescriptor(json: String): LibraryDescriptor = +private val jsonParser = Json { ignoreUnknownKeys = true } + +private inline fun parseCatching( + descriptor: T, + parse: (T) -> LibraryDescriptor, +): LibraryDescriptor = try { - Json.decodeFromString(json) + parse(descriptor) } catch (e: SerializationException) { - throw ReplException("Error during library deserialization. Library descriptor text:\n$json", e) + throw ReplException("Error during library deserialization. Library descriptor text:\n$descriptor", e) + } + +fun parseLibraryDescriptor(json: String): LibraryDescriptor = + parseCatching(json) { + jsonParser.decodeFromString(it) + } + +fun parseLibraryDescriptor(jsonObject: JsonObject): LibraryDescriptor = + parseCatching(jsonObject) { + jsonParser.decodeFromJsonElement(it) } fun parseLibraryDescriptors( diff --git a/src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/CustomLibraryResolverTests.kt b/src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/CustomLibraryResolverTests.kt index 990b5feb..47077f24 100644 --- a/src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/CustomLibraryResolverTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/CustomLibraryResolverTests.kt @@ -385,36 +385,6 @@ class CustomLibraryResolverTests : AbstractReplTest() { assertEquals(expected, repl.executedCodes) } - @Test - fun testIncorrectDescriptors() { - val ex1 = - assertThrows { - parseLibraryDescriptor( - """ - { - "imports": [] - """.trimIndent(), - ) - } - assertTrue(ex1.cause is SerializationException) - - val ex2 = - assertThrows { - parseLibraryDescriptor( - """ - { - "imports2": [] - } - """.trimIndent(), - ) - } - assertTrue(ex2.cause is SerializationException) - - assertDoesNotThrow { - parseLibraryDescriptor("{}") - } - } - @Test fun testLibraryWithResourcesDescriptorParsing() { val descriptor = parseLibraryDescriptor(File("src/test/testData/lib-with-resources.json").readText())