diff --git a/jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/compiler/util/serializedCompiledScript.kt b/jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/compiler/util/serializedCompiledScript.kt index 68742a461..ee792d312 100644 --- a/jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/compiler/util/serializedCompiledScript.kt +++ b/jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/compiler/util/serializedCompiledScript.kt @@ -20,9 +20,48 @@ data class SerializedCompiledScriptsData( } } +@Serializable +data class SerializableTypeInfo(val type: Type = Type.Custom, val isPrimitive: Boolean = false, val fullType: String = "") { + companion object { + val ignoreSet = setOf("int", "double", "boolean", "char", "float", "byte", "string", "entry") + + val propertyNamesForNullFilter = setOf("data", "size") + + fun makeFromSerializedVariablesState(type: String?, isContainer: Boolean?): SerializableTypeInfo { + val fullType = type.orEmpty() + val enumType = fullType.toTypeEnum() + val isPrimitive = !( + if (fullType != "Entry") (isContainer ?: false) + else true + ) + + return SerializableTypeInfo(enumType, isPrimitive, fullType) + } + } +} + +@Serializable +enum class Type { + Map, + Entry, + Array, + List, + Custom +} + +fun String.toTypeEnum(): Type { + return when (this) { + "Map" -> Type.Map + "Entry" -> Type.Entry + "Array" -> Type.Array + "List" -> Type.List + else -> Type.Custom + } +} + @Serializable data class SerializedVariablesState( - val type: String = "", + val type: SerializableTypeInfo = SerializableTypeInfo(), val value: String? = null, val isContainer: Boolean = false, val stateId: String = "" diff --git a/src/main/kotlin/org/jetbrains/kotlinx/jupyter/apiImpl.kt b/src/main/kotlin/org/jetbrains/kotlinx/jupyter/apiImpl.kt index 1364fb37a..c465ebe50 100644 --- a/src/main/kotlin/org/jetbrains/kotlinx/jupyter/apiImpl.kt +++ b/src/main/kotlin/org/jetbrains/kotlinx/jupyter/apiImpl.kt @@ -128,8 +128,9 @@ class NotebookImpl( private var currentCellVariables = mapOf>() private val history = arrayListOf() private var mainCellCreated = false - private val unchangedVariables: MutableSet = mutableSetOf() + private val _unchangedVariables: MutableSet = mutableSetOf() + val unchangedVariables: Set get() = _unchangedVariables val displays = DisplayContainerImpl() override fun getAllDisplays(): List { @@ -148,16 +149,14 @@ class NotebookImpl( fun updateVariablesState(evaluator: InternalEvaluator) { variablesState += evaluator.variablesHolder currentCellVariables = evaluator.cellVariables - unchangedVariables.clear() - unchangedVariables.addAll(evaluator.getUnchangedVariables()) + _unchangedVariables.clear() + _unchangedVariables.addAll(evaluator.getUnchangedVariables()) } fun updateVariablesState(varsStateUpdate: Map) { variablesState += varsStateUpdate } - fun unchangedVariables(): Set = unchangedVariables - fun variablesReportAsHTML(): String { return generateHTMLVarsReport(variablesState) } diff --git a/src/main/kotlin/org/jetbrains/kotlinx/jupyter/protocol.kt b/src/main/kotlin/org/jetbrains/kotlinx/jupyter/protocol.kt index 7d028e7a4..3462de942 100644 --- a/src/main/kotlin/org/jetbrains/kotlinx/jupyter/protocol.kt +++ b/src/main/kotlin/org/jetbrains/kotlinx/jupyter/protocol.kt @@ -2,8 +2,11 @@ package org.jetbrains.kotlinx.jupyter import ch.qos.logback.classic.Level import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.encodeToJsonElement +import kotlinx.serialization.json.jsonObject import org.jetbrains.annotations.TestOnly import org.jetbrains.kotlinx.jupyter.LoggingManagement.disableLogging import org.jetbrains.kotlinx.jupyter.LoggingManagement.mainLoggerLevel @@ -81,7 +84,6 @@ class OkResponseWithMessage( ) ) } - socket.send( makeReplyMessage( requestMsg, @@ -91,7 +93,7 @@ class OkResponseWithMessage( "engine" to Json.encodeToJsonElement(requestMsg.data.header?.session), "status" to Json.encodeToJsonElement("ok"), "started" to Json.encodeToJsonElement(startedTime), - "eval_metadata" to Json.encodeToJsonElement(metadata), + "eval_metadata" to Json.encodeToJsonElement(metadata.convertToNullIfEmpty()), ), content = ExecuteReply( MessageStatus.OK, @@ -316,7 +318,7 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup if (data.isEmpty()) return sendWrapped(msg, makeReplyMessage(msg, MessageType.SERIALIZATION_REPLY)) log.debug("Message data: $data") val messageContent = getVariablesDescriptorsFromJson(data) - GlobalScope.launch(Dispatchers.Default) { + connection.launchJob { repl.serializeVariables( messageContent.topLevelDescriptorName, messageContent.descriptorsState, @@ -342,7 +344,7 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup } } is SerializationRequest -> { - GlobalScope.launch(Dispatchers.Default) { + connection.launchJob { if (content.topLevelDescriptorName.isNotEmpty()) { repl.serializeVariables(content.topLevelDescriptorName, content.descriptorsState, commID = content.commId, content.pathToDescriptor) { result -> sendWrapped(msg, makeReplyMessage(msg, MessageType.SERIALIZATION_REPLY, content = result)) @@ -539,3 +541,8 @@ fun JupyterConnection.evalWithIO(repl: ReplForJupyter, srcMessage: Message, body KernelStreams.setStreams(false, out, err) } } + +fun EvaluatedSnippetMetadata?.convertToNullIfEmpty(): JsonElement? { + val jsonNode = Json.encodeToJsonElement(this) + return if (jsonNode is JsonNull || jsonNode?.jsonObject.isEmpty()) null else jsonNode +} diff --git a/src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl.kt b/src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl.kt index 75c190951..06c5b17e2 100644 --- a/src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl.kt +++ b/src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl.kt @@ -413,9 +413,10 @@ class ReplForJupyterImpl( val newImports: List val oldDeclarations: MutableMap = mutableMapOf() oldDeclarations.putAll(internalEvaluator.getVariablesDeclarationInfo()) + val jupyterId = evalData.jupyterId val result = try { - log.debug("Current cell id: ${evalData.jupyterId}") - executor.execute(evalData.code, evalData.displayHandler, currentCellId = evalData.jupyterId - 1) { internalId, codeToExecute -> + log.debug("Current cell id: $jupyterId") + executor.execute(evalData.code, evalData.displayHandler, currentCellId = jupyterId - 1) { internalId, codeToExecute -> if (evalData.storeHistory) { cell = notebook.addCell(internalId, codeToExecute, EvalData(evalData)) } @@ -444,7 +445,7 @@ class ReplForJupyterImpl( // printVars() // printUsagesInfo(jupyterId, cellVariables[jupyterId - 1]) val variablesCells: Map = notebook.variablesState.mapValues { internalEvaluator.findVariableCell(it.key) } - val serializedData = variablesSerializer.serializeVariables(jupyterId - 1, notebook.variablesState, oldDeclarations, variablesCells, notebook.unchangedVariables()) + val serializedData = variablesSerializer.serializeVariables(jupyterId - 1, notebook.variablesState, oldDeclarations, variablesCells, notebook.unchangedVariables) GlobalScope.launch(Dispatchers.Default) { variablesSerializer.tryValidateCache(jupyterId - 1, notebook.cellVariables) diff --git a/src/main/kotlin/org/jetbrains/kotlinx/jupyter/serializationUtils.kt b/src/main/kotlin/org/jetbrains/kotlinx/jupyter/serializationUtils.kt index 1ba2d1409..91f745518 100644 --- a/src/main/kotlin/org/jetbrains/kotlinx/jupyter/serializationUtils.kt +++ b/src/main/kotlin/org/jetbrains/kotlinx/jupyter/serializationUtils.kt @@ -5,6 +5,7 @@ import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.decodeFromJsonElement import org.jetbrains.kotlinx.jupyter.api.VariableState +import org.jetbrains.kotlinx.jupyter.compiler.util.SerializableTypeInfo import org.jetbrains.kotlinx.jupyter.compiler.util.SerializedVariablesState import java.lang.reflect.Field import kotlin.contracts.ExperimentalContracts @@ -32,14 +33,14 @@ enum class PropertiesType { } @Serializable -data class SerializedCommMessageContent( +data class VariablesStateCommMessageContent( val topLevelDescriptorName: String, val descriptorsState: Map, val pathToDescriptor: List = emptyList() ) -fun getVariablesDescriptorsFromJson(json: JsonObject): SerializedCommMessageContent { - return Json.decodeFromJsonElement(json) +fun getVariablesDescriptorsFromJson(json: JsonObject): VariablesStateCommMessageContent { + return Json.decodeFromJsonElement(json) } class ProcessedSerializedVarsState( @@ -216,7 +217,12 @@ class VariablesSerializer( } else { "" } - val serializedVersion = SerializedVariablesState(simpleTypeName, stringedValue, true, varID) + val serializedVersion = SerializedVariablesState( + SerializableTypeInfo.makeFromSerializedVariablesState(simpleTypeName, true), + stringedValue, + true, + varID + ) val descriptors = serializedVersion.fieldDescriptor // only for set case @@ -700,7 +706,12 @@ class VariablesSerializer( "" } - val serializedVariablesState = SerializedVariablesState(type, getProperString(value), isContainer, finalID) + val serializedVariablesState = SerializedVariablesState( + SerializableTypeInfo.makeFromSerializedVariablesState(simpleTypeName, isContainer), + getProperString(value), + isContainer, + finalID + ) return ProcessedSerializedVarsState(serializedVariablesState, membersProperties?.toTypedArray()) } diff --git a/src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/ReplTests.kt b/src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/ReplTests.kt index eaf957dfe..f6cc8fcb9 100644 --- a/src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/ReplTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/ReplTests.kt @@ -26,8 +26,8 @@ import kotlin.test.assertEquals import kotlin.test.assertFails import kotlin.test.assertFailsWith import kotlin.test.assertFalse -import kotlin.test.assertNotNull import kotlin.test.assertNotEquals +import kotlin.test.assertNotNull import kotlin.test.fail class ReplTests : AbstractSingleReplTest() { @@ -834,14 +834,13 @@ class ReplVarsTest : AbstractSingleReplTest() { """.trimIndent(), jupyterId = 1 ) - var state = repl.notebook.unchangedVariables() - val res = eval( + eval( """ l += 11111 """.trimIndent(), jupyterId = 2 ).metadata.evaluatedVariablesState - state = repl.notebook.unchangedVariables() + val state: Set = repl.notebook.unchangedVariables assertEquals(1, state.size) assertTrue(state.contains("m")) } @@ -911,7 +910,7 @@ class ReplVarsTest : AbstractSingleReplTest() { """.trimIndent(), jupyterId = 1 ) - var state = repl.notebook.unchangedVariables() + var state = repl.notebook.unchangedVariables assertEquals(3, state.size) eval( @@ -922,7 +921,7 @@ class ReplVarsTest : AbstractSingleReplTest() { """.trimIndent(), jupyterId = 2 ) - state = repl.notebook.unchangedVariables() + state = repl.notebook.unchangedVariables assertEquals(0, state.size) eval( @@ -931,7 +930,7 @@ class ReplVarsTest : AbstractSingleReplTest() { """.trimIndent(), jupyterId = 3 ) - state = repl.notebook.unchangedVariables() + state = repl.notebook.unchangedVariables assertEquals(1, state.size) } } @@ -967,7 +966,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() { assertEquals(listOf(1, 2, 3, 4).toString().substring(1, actualContainer.value!!.length + 1), actualContainer.value) val serializer = repl.variablesSerializer - val newData = serializer.doIncrementalSerialization(0, "x", "data", actualContainer) + serializer.doIncrementalSerialization(0, "x", "data", actualContainer) } @Test @@ -983,7 +982,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() { assertEquals(2, varsData.size) assertTrue(varsData.containsKey("x")) assertTrue(varsData.containsKey("f")) - var unchangedVariables = repl.notebook.unchangedVariables() + var unchangedVariables = repl.notebook.unchangedVariables assertTrue(unchangedVariables.isNotEmpty()) eval( @@ -992,7 +991,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() { """.trimIndent(), jupyterId = 1 ) - unchangedVariables = repl.notebook.unchangedVariables() + unchangedVariables = repl.notebook.unchangedVariables assertTrue(unchangedVariables.contains("x")) assertTrue(unchangedVariables.contains("f")) } @@ -1056,7 +1055,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() { val serializer = repl.variablesSerializer - val newData = serializer.doIncrementalSerialization(0, "c", "i", descriptor["i"]!!) + serializer.doIncrementalSerialization(0, "c", "i", descriptor["i"]!!) } @Test @@ -1345,7 +1344,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() { """.trimIndent(), jupyterId = 1 ) - val state = repl.notebook.unchangedVariables() + val state = repl.notebook.unchangedVariables val setOfCell = setOf("x", "f", "z") assertTrue(state.isNotEmpty()) assertEquals(setOfCell, state) @@ -1372,7 +1371,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() { """.trimIndent(), jupyterId = 1 ) - var state = repl.notebook.unchangedVariables() + var state = repl.notebook.unchangedVariables val setOfCell = setOf("x", "f", "z") assertTrue(state.isNotEmpty()) assertEquals(setOfCell, state) @@ -1396,9 +1395,9 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() { """.trimIndent(), jupyterId = 3 ) - state = repl.notebook.unchangedVariables() -// assertTrue(state.isNotEmpty()) -// assertEquals(state, setOfPrevCell) + state = repl.notebook.unchangedVariables + assertTrue(state.isEmpty()) + // assertEquals(state, setOfPrevCell) eval( """ @@ -1408,20 +1407,20 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() { """.trimIndent(), jupyterId = 4 ) - state = repl.notebook.unchangedVariables() + state = repl.notebook.unchangedVariables assertTrue(state.isEmpty()) } @Test fun testSerializationClearInfo() { - var res = eval( + eval( """ val x = listOf(1, 2, 3, 4) """.trimIndent(), jupyterId = 1 ).metadata.evaluatedVariablesState - var state = repl.notebook.unchangedVariables() - res = eval( + repl.notebook.unchangedVariables + eval( """ val x = listOf(1, 2, 3, 4) """.trimIndent(),