Descriptors in libraries.json

docs/libraries.md

@@ -933,13 +933,39 @@ resources. This file should contain FQNs of all integration classes in the JSON
         {
             "fqn": "org.jetbrains.kotlinx.jupyter.example.GettingStartedIntegration"
         }
-    ]
+    ],
+    "descriptors": []
 }
 ```
 
 Classes derived from the `LibraryDefinition` interface should be added to the `definitions` array.
 Classes derived from the `LibraryDefinitionProducer` interface should be added to the `producers` array.
 
+Additionally, you're allowed to add full custom library descriptors to the `descriptors` array.
+This is an advanced option that allows you to load additional dependencies or add additional imports using the
+[library descriptor API](#creating-a-library-descriptor).
+This can be particularly useful for libraries that cannot depend on the jupyter-api artifact but still want to modify
+the kernel behavior when loaded.
+
+For example:
+
+```json
+{
+    "definitions": [], 
+    "producers": [],
+    "descriptors": [
+      {
+        "dependencies": [
+          "my.library:my-library-jupyter-integration:1.0.0"
+        ],
+        "init": [
+          "DISPLAY(\"Implicitly loading Jupyter integration for MyLibrary...\")"
+        ]
+      }
+    ]
+}
+```
+
 For more information, see:
 
 * [Libraries repository](https://github.com/Kotlin/kotlin-jupyter-libraries)

jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/libraries/Scanning.kt

@@ -1,6 +1,7 @@
 package org.jetbrains.kotlinx.jupyter.api.libraries
 
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.JsonObject
 import org.jetbrains.kotlinx.jupyter.api.TypeName
 
 /**
@@ -48,9 +49,13 @@ data class LibrariesProducerDeclaration(
 /**
  * Serialized form of this class instance is a correct content of
  * [KOTLIN_JUPYTER_LIBRARIES_FILE_NAME] file, and vice versa.
+ *
+ * @param descriptors List of library descriptor JSON objects adhering to
+ *   `org.jetbrains.kotlinx.jupyter.libraries.LibraryDescriptor`.
  */
 @Serializable
 data class LibrariesScanResult(
     val definitions: List<LibrariesDefinitionDeclaration> = emptyList(),
     val producers: List<LibrariesProducerDeclaration> = emptyList(),
+    val descriptors: List<JsonObject> = emptyList(),
 )

jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/ApiGradlePlugin.kt

@@ -12,17 +12,13 @@ import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
 import org.jetbrains.kotlin.gradle.targets.jvm.KotlinJvmTarget
 import org.jetbrains.kotlinx.jupyter.api.plugin.tasks.JupyterApiResourcesTask
 import org.jetbrains.kotlinx.jupyter.api.plugin.util.addMavenCentralIfDoesNotExist
-import org.jetbrains.kotlinx.jupyter.api.plugin.util.getBuildDirectory
 import org.jetbrains.kotlinx.jupyter.api.plugin.util.whenAdded
 
 class ApiGradlePlugin : Plugin<Project> {
     override fun apply(target: Project): Unit =
         with(target) {
             val pluginExtension = KotlinJupyterPluginExtension(target)
             extensions.add(KotlinJupyterPluginExtension.NAME, pluginExtension)
-
-            val jupyterBuildPath = getBuildDirectory().resolve(FQNS_PATH)
-            jupyterBuildPath.mkdirs()
             pluginExtension.addDependenciesIfNeeded()
 
             repositories {
@@ -63,8 +59,4 @@ class ApiGradlePlugin : Plugin<Project> {
                 }
             }
         }
-
-    companion object {
-        const val FQNS_PATH = "generated/jupyter/fqns"
-    }
 }

jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/KotlinJupyterPluginExtension.kt

@@ -1,6 +1,7 @@
 package org.jetbrains.kotlinx.jupyter.api.plugin
 
 import org.gradle.api.Project
+import org.intellij.lang.annotations.Language
 import org.jetbrains.kotlinx.jupyter.api.plugin.util.FQNAware
 import org.jetbrains.kotlinx.jupyter.api.plugin.util.LibrariesScanResult
 import org.jetbrains.kotlinx.jupyter.api.plugin.util.configureDependency
@@ -15,11 +16,13 @@ class KotlinJupyterPluginExtension(
 
     private val libraryProducers: MutableSet<FQNAware> = mutableSetOf()
     private val libraryDefinitions: MutableSet<FQNAware> = mutableSetOf()
+    private val libraryDescriptors: MutableSet<String> = mutableSetOf()
 
     internal val libraryFqns get() =
         LibrariesScanResult(
             definitions = libraryDefinitions,
             producers = libraryProducers,
+            descriptors = libraryDescriptors,
         )
 
     internal fun addDependenciesIfNeeded() {
@@ -42,6 +45,7 @@ class KotlinJupyterPluginExtension(
     /**
      * Add adding library integrations by specifying their fully qualified names
      */
+    @Suppress("unused")
     fun integrations(action: IntegrationsSpec.() -> Unit) {
         IntegrationsSpec().apply(action)
     }
@@ -56,6 +60,13 @@ class KotlinJupyterPluginExtension(
         fun definition(className: String) {
             libraryDefinitions.add(FQNAware(className))
         }
+
+        @Suppress("unused")
+        fun descriptor(
+            @Language("JSON") descriptor: String,
+        ) {
+            libraryDescriptors.add(descriptor)
+        }
     }
 
     companion object {

jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/tasks/JupyterApiResourcesTask.kt

@@ -5,7 +5,6 @@ import org.gradle.api.DefaultTask
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.OutputDirectory
 import org.gradle.api.tasks.TaskAction
-import org.jetbrains.kotlinx.jupyter.api.plugin.ApiGradlePlugin
 import org.jetbrains.kotlinx.jupyter.api.plugin.KotlinJupyterPluginExtension
 import org.jetbrains.kotlinx.jupyter.api.plugin.util.FQNAware
 import org.jetbrains.kotlinx.jupyter.api.plugin.util.LibrariesScanResult
@@ -30,6 +29,13 @@ open class JupyterApiResourcesTask : DefaultTask() {
     @Input
     var libraryDefinitions: List<String> = emptyList()
 
+    /**
+     * List of JSON library descriptors in the
+     * [library descriptor format](https://github.com/Kotlin/kotlin-jupyter/blob/master/docs/libraries.md#creating-a-library-descriptor).
+     */
+    @Input
+    var libraryDescriptors: List<String> = emptyList()
+
     @OutputDirectory
     val outputDir: File = project.getBuildDirectory().resolve("jupyterProcessedResources")
 
@@ -42,12 +48,10 @@ open class JupyterApiResourcesTask : DefaultTask() {
             LibrariesScanResult(
                 definitions = libraryDefinitions.map { FQNAware(it) }.toSet(),
                 producers = libraryProducers.map { FQNAware(it) }.toSet(),
+                descriptors = libraryDescriptors.toSet(),
             )
 
-        val resultObject =
-            jupyterExtensionScanResult +
-                taskScanResult +
-                getScanResultFromAnnotations()
+        val resultObject = jupyterExtensionScanResult + taskScanResult
         val json = Gson().toJson(resultObject)
 
         val jupyterDir = outputDir.resolve("META-INF/kotlin-jupyter-libraries")
@@ -56,28 +60,10 @@ open class JupyterApiResourcesTask : DefaultTask() {
         libFile.writeText(json)
     }
 
-    private fun getScanResultFromAnnotations(): LibrariesScanResult {
-        val path = project.getBuildDirectory().resolve(ApiGradlePlugin.FQNS_PATH)
-
-        fun fqns(name: String): Set<FQNAware> {
-            val file = path.resolve(name)
-            if (!file.exists()) return emptySet()
-            return file
-                .readLines()
-                .filter { it.isNotBlank() }
-                .map { FQNAware(it) }
-                .toSet()
-        }
-
-        return LibrariesScanResult(
-            fqns("definitions"),
-            fqns("producers"),
-        )
-    }
-
     operator fun LibrariesScanResult.plus(other: LibrariesScanResult): LibrariesScanResult =
         LibrariesScanResult(
-            definitions + other.definitions,
-            producers + other.producers,
+            definitions = definitions + other.definitions,
+            producers = producers + other.producers,
+            descriptors = descriptors + other.descriptors,
         )
 }

jupyter-lib/kotlin-jupyter-api-gradle-plugin/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/plugin/util/LibrariesUtil.kt

@@ -7,6 +7,7 @@ data class FQNAware(
 class LibrariesScanResult(
     val definitions: Set<FQNAware> = emptySet(),
     val producers: Set<FQNAware> = emptySet(),
+    val descriptors: Set<String> = emptySet(),
 )
 
 val emptyScanResult = LibrariesScanResult()

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/LibrariesScanner.kt

@@ -1,6 +1,7 @@
 package org.jetbrains.kotlinx.jupyter.libraries
 
 import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
 import org.jetbrains.kotlinx.jupyter.api.KotlinKernelHost
 import org.jetbrains.kotlinx.jupyter.api.LibraryLoader
 import org.jetbrains.kotlinx.jupyter.api.Notebook
@@ -13,6 +14,7 @@ import org.jetbrains.kotlinx.jupyter.api.libraries.LibrariesInstantiable
 import org.jetbrains.kotlinx.jupyter.api.libraries.LibrariesProducerDeclaration
 import org.jetbrains.kotlinx.jupyter.api.libraries.LibrariesScanResult
 import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinition
+import org.jetbrains.kotlinx.jupyter.api.libraries.Variable
 import org.jetbrains.kotlinx.jupyter.config.errorForUser
 import org.jetbrains.kotlinx.jupyter.protocol.api.KernelLoggerFactory
 import org.jetbrains.kotlinx.jupyter.protocol.api.getLogger
@@ -24,9 +26,11 @@ class LibrariesScanner(
     loggerFactory: KernelLoggerFactory,
 ) : LibraryLoader {
     private val logger = loggerFactory.getLogger(this::class)
+    private val jsonParser = Json { ignoreUnknownKeys = true }
 
     private val processedFQNs = mutableSetOf<TypeName>()
     private val discardedFQNs = mutableSetOf<TypeName>()
+    private val processedDescriptorHashes = mutableSetOf<Int>()
 
     private fun <I : LibrariesInstantiable<*>> Iterable<I>.filterNamesToLoad(
         host: KotlinKernelHost,
@@ -46,10 +50,14 @@ class LibrariesScanner(
                     discardedFQNs.add(typeName)
                     false
                 }
+
                 null -> typeName !in discardedFQNs && processedFQNs.add(typeName)
             }
         }
 
+    /** Makes sure each unique descriptor is only loaded once by caching their hashes in [processedDescriptorHashes]. */
+    private fun Iterable<JsonObject>.filterDescriptorsToLoad() = filter { processedDescriptorHashes.add(it.hashCode()) }
+
     fun addLibrariesFromClassLoader(
         classLoader: ClassLoader,
         host: KotlinKernelHost,
@@ -83,24 +91,30 @@ class LibrariesScanner(
         integrationTypeNameRules: List<AcceptanceRule<TypeName>> = listOf(),
     ): LibrariesScanResult {
         val results =
-            classLoader.getResources("$KOTLIN_JUPYTER_RESOURCES_PATH/$KOTLIN_JUPYTER_LIBRARIES_FILE_NAME").toList().map { url ->
-                val contents = url.readText()
-                Json.decodeFromString<LibrariesScanResult>(contents)
-            }
+            classLoader
+                .getResources("$KOTLIN_JUPYTER_RESOURCES_PATH/$KOTLIN_JUPYTER_LIBRARIES_FILE_NAME")
+                .toList()
+                .map { url ->
+                    val contents = url.readText()
+                    jsonParser.decodeFromString<LibrariesScanResult>(contents)
+                }
 
         val definitions = mutableListOf<LibrariesDefinitionDeclaration>()
         val producers = mutableListOf<LibrariesProducerDeclaration>()
+        val descriptors = mutableListOf<JsonObject>()
 
         for (result in results) {
             definitions.addAll(result.definitions)
             producers.addAll(result.producers)
+            descriptors.addAll(result.descriptors)
         }
 
         fun <I : LibrariesInstantiable<*>> Iterable<I>.filterNames() = filterNamesToLoad(host, integrationTypeNameRules)
 
         return LibrariesScanResult(
             definitions.filterNames(),
             producers.filterNames(),
+            descriptors.filterDescriptorsToLoad(),
         )
     }
 
@@ -140,6 +154,14 @@ class LibrariesScanner(
                 }
             }
         }
+
+        scanResult.descriptors.forEach {
+            val descriptor = parseLibraryDescriptor(it)
+            val arguments = libraryOptions.map { (name, value) -> Variable(name, value) }
+            val definition = descriptor.convertToDefinition(arguments)
+            definitions.add(definition)
+        }
+
         return definitions
     }
 

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/Parsing.kt

@@ -2,14 +2,31 @@ package org.jetbrains.kotlinx.jupyter.libraries
 
 import kotlinx.serialization.SerializationException
 import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.decodeFromJsonElement
 import org.jetbrains.kotlinx.jupyter.api.exceptions.ReplException
 import org.jetbrains.kotlinx.jupyter.protocol.api.KernelLoggerFactory
 
-fun parseLibraryDescriptor(json: String): LibraryDescriptor =
+private val jsonParser = Json { ignoreUnknownKeys = true }
+
+private inline fun <T> parseCatching(
+    descriptor: T,
+    parse: (T) -> LibraryDescriptor,
+): LibraryDescriptor =
     try {
-        Json.decodeFromString(json)
+        parse(descriptor)
     } catch (e: SerializationException) {
-        throw ReplException("Error during library deserialization. Library descriptor text:\n$json", e)
+        throw ReplException("Error during library deserialization. Library descriptor text:\n$descriptor", e)
+    }
+
+fun parseLibraryDescriptor(json: String): LibraryDescriptor =
+    parseCatching(json) {
+        jsonParser.decodeFromString(it)
+    }
+
+fun parseLibraryDescriptor(jsonObject: JsonObject): LibraryDescriptor =
+    parseCatching(jsonObject) {
+        jsonParser.decodeFromJsonElement(it)
     }
 
 fun parseLibraryDescriptors(

src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/CustomLibraryResolverTests.kt

@@ -385,36 +385,6 @@ class CustomLibraryResolverTests : AbstractReplTest() {
         assertEquals(expected, repl.executedCodes)
     }
 
-    @Test
-    fun testIncorrectDescriptors() {
-        val ex1 =
-            assertThrows<ReplException> {
-                parseLibraryDescriptor(
-                    """
-                    {
-                        "imports": []
-                    """.trimIndent(),
-                )
-            }
-        assertTrue(ex1.cause is SerializationException)
-
-        val ex2 =
-            assertThrows<ReplException> {
-                parseLibraryDescriptor(
-                    """
-                    {
-                        "imports2": []
-                    }
-                    """.trimIndent(),
-                )
-            }
-        assertTrue(ex2.cause is SerializationException)
-
-        assertDoesNotThrow {
-            parseLibraryDescriptor("{}")
-        }
-    }
-
     @Test
     fun testLibraryWithResourcesDescriptorParsing() {
         val descriptor = parseLibraryDescriptor(File("src/test/testData/lib-with-resources.json").readText())