fix: live variable decoding error on Python dicts

marker/py-marker/build.gradle.kts

@@ -21,6 +21,7 @@ dependencies {
     compileOnly("org.jetbrains:annotations:23.0.0")
     compileOnly("io.vertx:vertx-core:$vertxVersion")
 
+    testImplementation("io.vertx:vertx-core:$vertxVersion")
     testImplementation(projectDependency(":common"))
     testImplementation("org.junit.jupiter:junit-jupiter:$jupiterVersion")
 }

marker/py-marker/src/main/kotlin/spp/jetbrains/marker/py/presentation/PythonVariableNode.kt

@@ -45,10 +45,7 @@ class PythonVariableNode(
 
     override fun getChildren(): Array<SimpleNode> {
         if (variable.liveClazz == "<class 'dict'>") {
-            val dict = JsonObject(
-                (variable.value as String).replace("'", "\"")
-                    .replace(": True", ": true").replace(": False", ": false")
-            )
+            val dict = JsonObject(parseDict(variable.value as String))
             val children = mutableListOf<SimpleNode>()
             dict.map.forEach {
                 children.add(PythonVariableNode(LiveVariable("'" + it.key + "'", it.value)))
@@ -101,4 +98,56 @@ class PythonVariableNode(
             }
         }
     }
+
+    companion object {
+        fun parseDict(value: String): Map<String, String> {
+            var dictVar = value
+
+            //remove outer braces
+            if (dictVar.startsWith("{")) {
+                dictVar = dictVar.substring(1)
+            }
+            if (dictVar.endsWith("}")) {
+                dictVar = dictVar.substring(0, dictVar.length - 1)
+            }
+
+            //split by , but not inside quotes
+            val tokens = dictVar.split(Regex(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)"))
+
+            //each token should contain ":", for those that don't we need to merge them with the previous token
+            val mergedTokens = mutableListOf<String>()
+            for (i in tokens.indices) {
+                if (tokens[i].contains(":")) {
+                    mergedTokens.add(tokens[i])
+                } else {
+                    mergedTokens[mergedTokens.size - 1] += "," + tokens[i]
+                }
+            }
+
+            //now we can split each token into key and value
+            val dict = mutableMapOf<String, String>()
+            for (token in mergedTokens) {
+                var key = token.substringBefore(":").trim()
+                var value = token.substringAfter(":").trim()
+
+                //remove quotes
+                if (key.startsWith("\"")) {
+                    key = key.substring(1)
+                }
+                if (key.endsWith("\"")) {
+                    key = key.substring(0, key.length - 1)
+                }
+                if (value.startsWith("\"")) {
+                    value = value.substring(1)
+                }
+                if (value.endsWith("\"")) {
+                    value = value.substring(0, value.length - 1)
+                }
+
+                dict[key] = value
+            }
+
+            return dict
+        }
+    }
 }

marker/py-marker/src/test/kotlin/spp/jetbrains/marker/py/presentation/LiveVariableParseTest.kt

@@ -0,0 +1,39 @@
+/*
+ * Source++, the open-source live coding platform.
+ * Copyright (C) 2022 CodeBrig, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package spp.jetbrains.marker.py.presentation
+
+import com.google.common.io.Resources
+import io.vertx.core.json.JsonObject
+import org.junit.Test
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertNotNull
+import spp.protocol.instrument.event.LiveBreakpointHit
+
+class LiveVariableParseTest {
+
+    @Test
+    fun testDictParse() {
+        val bpHitJson = Resources.getResource("breakpointHit/dictParse.json").readText()
+        val bpHit = LiveBreakpointHit(JsonObject(bpHitJson))
+        val builtInsVar = bpHit.stackTrace.elements.first().variables.find { it.name == "__builtins__" }
+        assertNotNull(builtInsVar)
+
+        val parsedDict = PythonVariableNode.parseDict(builtInsVar!!.value as String)
+        assertNotNull(parsedDict)
+        assertEquals(152, parsedDict.size)
+    }
+}

marker/py-marker/src/test/resources/breakpointHit/dictParse.json

@@ -0,0 +1,167 @@
+{
+  "breakpointId": "914439a3-863b-40b5-8911-2519ba697413",
+  "traceId": "cdb70afe43e411ed94590242c0a8900b",
+  "occurredAt": "2022-10-04T13:02:30.128Z",
+  "serviceInstance": "9f7f48ec43dd11ed94410242c0a8900b",
+  "service": "demo-python",
+  "stackTrace": {
+    "exceptionType": "n/a",
+    "message": "n/a",
+    "elements": [
+      {
+        "method": "simple_breakpoint",
+        "source": "/demo-python/src/command/AddBreakpoint.py:30",
+        "column": null,
+        "variables": [
+          {
+            "name": "__name__",
+            "value": "command.AddBreakpoint",
+            "lineNumber": -1,
+            "scope": "GLOBAL_VARIABLE",
+            "liveClazz": "<class 'str'>",
+            "liveIdentity": "140259926832512",
+            "presentation": null
+          },
+          {
+            "name": "__doc__",
+            "value": "None",
+            "lineNumber": -1,
+            "scope": "GLOBAL_VARIABLE",
+            "liveClazz": "<class 'NoneType'>",
+            "liveIdentity": "140259935552032",
+            "presentation": null
+          },
+          {
+            "name": "__package__",
+            "value": "command",
+            "lineNumber": -1,
+            "scope": "GLOBAL_VARIABLE",
+            "liveClazz": "<class 'str'>",
+            "liveIdentity": "140259851025840",
+            "presentation": null
+          },
+          {
+            "name": "__loader__",
+            "value": "<_frozen_importlib_external.SourceFileLoader object at 0x7f90ca97e340>",
+            "lineNumber": -1,
+            "scope": "GLOBAL_VARIABLE",
+            "liveClazz": "<class '_frozen_importlib_external.SourceFileLoader'>",
+            "liveIdentity": "140259850969920",
+            "presentation": null
+          },
+          {
+            "name": "__spec__",
+            "value": "ModuleSpec(name='command.AddBreakpoint', loader=<_frozen_importlib_external.SourceFileLoader object at 0x7f90ca97e340>, origin='/demo-python/src/command/AddBreakpoint.py')",
+            "lineNumber": -1,
+            "scope": "GLOBAL_VARIABLE",
+            "liveClazz": "<class '_frozen_importlib.ModuleSpec'>",
+            "liveIdentity": "140259850971552",
+            "presentation": null
+          },
+          {
+            "name": "__file__",
+            "value": "/demo-python/src/command/AddBreakpoint.py",
+            "lineNumber": -1,
+            "scope": "GLOBAL_VARIABLE",
+            "liveClazz": "<class 'str'>",
+            "liveIdentity": "140259850966096",
+            "presentation": null
+          },
+          {
+            "name": "__cached__",
+            "value": "/demo-python/src/command/__pycache__/AddBreakpoint.cpython-38.pyc",
+            "lineNumber": -1,
+            "scope": "GLOBAL_VARIABLE",
+            "liveClazz": "<class 'str'>",
+            "liveIdentity": "140259927272240",
+            "presentation": null
+          },
+          {
+            "name": "__builtins__",
+            "value": "{'__name__': 'builtins', '__doc__': \"Built-in functions, exceptions, and other objects.\\n\\nNoteworthy: None is the `nil' object; Ellipsis represents `...' in slices.\", '__package__': '', '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': ModuleSpec(name='builtins', loader=<class '_frozen_importlib.BuiltinImporter'>), '__build_class__': <built-in function __build_class__>, '__import__': <built-in function __import__>, 'abs': <built-in function abs>, 'all': <built-in function all>, 'any': <built-in function any>, 'ascii': <built-in function ascii>, 'bin': <built-in function bin>, 'breakpoint': <built-in function breakpoint>, 'callable': <built-in function callable>, 'chr': <built-in function chr>, 'compile': <built-in function compile>, 'delattr': <built-in function delattr>, 'dir': <built-in function dir>, 'divmod': <built-in function divmod>, 'eval': <built-in function eval>, 'exec': <built-in function exec>, 'format': <built-in function format>, 'getattr': <built-in function getattr>, 'globals': <built-in function globals>, 'hasattr': <built-in function hasattr>, 'hash': <built-in function hash>, 'hex': <built-in function hex>, 'id': <built-in function id>, 'input': <built-in function input>, 'isinstance': <built-in function isinstance>, 'issubclass': <built-in function issubclass>, 'iter': <built-in function iter>, 'len': <built-in function len>, 'locals': <built-in function locals>, 'max': <built-in function max>, 'min': <built-in function min>, 'next': <built-in function next>, 'oct': <built-in function oct>, 'ord': <built-in function ord>, 'pow': <built-in function pow>, 'print': <built-in function print>, 'repr': <built-in function repr>, 'round': <built-in function round>, 'setattr': <built-in function setattr>, 'sorted': <built-in function sorted>, 'sum': <built-in function sum>, 'vars': <built-in function vars>, 'None': None, 'Ellipsis': Ellipsis, 'NotImplemented': NotImplemented, 'False': False, 'True': True, 'bool': <class 'bool'>, 'memoryview': <class 'memoryview'>, 'bytearray': <class 'bytearray'>, 'bytes': <class 'bytes'>, 'classmethod': <class 'classmethod'>, 'complex': <class 'complex'>, 'dict': <class 'dict'>, 'enumerate': <class 'enumerate'>, 'filter': <class 'filter'>, 'float': <class 'float'>, 'frozenset': <class 'frozenset'>, 'property': <class 'property'>, 'int': <class 'int'>, 'list': <class 'list'>, 'map': <class 'map'>, 'object': <class 'object'>, 'range': <class 'range'>, 'reversed': <class 'reversed'>, 'set': <class 'set'>, 'slice': <class 'slice'>, 'staticmethod': <class 'staticmethod'>, 'str': <class 'str'>, 'super': <class 'super'>, 'tuple': <class 'tuple'>, 'type': <class 'type'>, 'zip': <class 'zip'>, '__debug__': True, 'BaseException': <class 'BaseException'>, 'Exception': <class 'Exception'>, 'TypeError': <class 'TypeError'>, 'StopAsyncIteration': <class 'StopAsyncIteration'>, 'StopIteration': <class 'StopIteration'>, 'GeneratorExit': <class 'GeneratorExit'>, 'SystemExit': <class 'SystemExit'>, 'KeyboardInterrupt': <class 'KeyboardInterrupt'>, 'ImportError': <class 'ImportError'>, 'ModuleNotFoundError': <class 'ModuleNotFoundError'>, 'OSError': <class 'OSError'>, 'EnvironmentError': <class 'OSError'>, 'IOError': <class 'OSError'>, 'EOFError': <class 'EOFError'>, 'RuntimeError': <class 'RuntimeError'>, 'RecursionError': <class 'RecursionError'>, 'NotImplementedError': <class 'NotImplementedError'>, 'NameError': <class 'NameError'>, 'UnboundLocalError': <class 'UnboundLocalError'>, 'AttributeError': <class 'AttributeError'>, 'SyntaxError': <class 'SyntaxError'>, 'IndentationError': <class 'IndentationError'>, 'TabError': <class 'TabError'>, 'LookupError': <class 'LookupError'>, 'IndexError': <class 'IndexError'>, 'KeyError': <class 'KeyError'>, 'ValueError': <class 'ValueError'>, 'UnicodeError': <class 'UnicodeError'>, 'UnicodeEncodeError': <class 'UnicodeEncodeError'>, 'UnicodeDecodeError': <class 'UnicodeDecodeError'>, 'UnicodeTranslateError': <class 'UnicodeTranslateError'>, 'AssertionError': <class 'AssertionError'>, 'ArithmeticError': <class 'ArithmeticError'>, 'FloatingPointError': <class 'FloatingPointError'>, 'OverflowError': <class 'OverflowError'>, 'ZeroDivisionError': <class 'ZeroDivisionError'>, 'SystemError': <class 'SystemError'>, 'ReferenceError': <class 'ReferenceError'>, 'MemoryError': <class 'MemoryError'>, 'BufferError': <class 'BufferError'>, 'Warning': <class 'Warning'>, 'UserWarning': <class 'UserWarning'>, 'DeprecationWarning': <class 'DeprecationWarning'>, 'PendingDeprecationWarning': <class 'PendingDeprecationWarning'>, 'SyntaxWarning': <class 'SyntaxWarning'>, 'RuntimeWarning': <class 'RuntimeWarning'>, 'FutureWarning': <class 'FutureWarning'>, 'ImportWarning': <class 'ImportWarning'>, 'UnicodeWarning': <class 'UnicodeWarning'>, 'BytesWarning': <class 'BytesWarning'>, 'ResourceWarning': <class 'ResourceWarning'>, 'ConnectionError': <class 'ConnectionError'>, 'BlockingIOError': <class 'BlockingIOError'>, 'BrokenPipeError': <class 'BrokenPipeError'>, 'ChildProcessError': <class 'ChildProcessError'>, 'ConnectionAbortedError': <class 'ConnectionAbortedError'>, 'ConnectionRefusedError': <class 'ConnectionRefusedError'>, 'ConnectionResetError': <class 'ConnectionResetError'>, 'FileExistsError': <class 'FileExistsError'>, 'FileNotFoundError': <class 'FileNotFoundError'>, 'IsADirectoryError': <class 'IsADirectoryError'>, 'NotADirectoryError': <class 'NotADirectoryError'>, 'InterruptedError': <class 'InterruptedError'>, 'PermissionError': <class 'PermissionError'>, 'ProcessLookupError': <class 'ProcessLookupError'>, 'TimeoutError': <class 'TimeoutError'>, 'open': <built-in function open>, 'quit': Use quit() or Ctrl-D (i.e. EOF) to exit, 'exit': Use exit() or Ctrl-D (i.e. EOF) to exit, 'copyright': Copyright (c) 2001-2022 Python Software Foundation.\nAll Rights Reserved.\n\nCopyright (c) 2000 BeOpen.com.\nAll Rights Reserved.\n\nCopyright (c) 1995-2001 Corporation for National Research Initiatives.\nAll Rights Reserved.\n\nCopyright (c) 1991-1995 Stichting Mathematisch Centrum, Amsterdam.\nAll Rights Reserved., 'credits':     Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands\n    for supporting Python development.  See www.python.org for more information., 'license': Type license() to see the full license text, 'help': Type help() for interactive help, or help(object) for help about object.}",
+            "lineNumber": -1,
+            "scope": "GLOBAL_VARIABLE",
+            "liveClazz": "<class 'dict'>",
+            "liveIdentity": "140259928018176",
+            "presentation": null
+          },
+          {
+            "name": "AddBreakpoint",
+            "value": "<class 'command.AddBreakpoint.AddBreakpoint'>",
+            "lineNumber": -1,
+            "scope": "GLOBAL_VARIABLE",
+            "liveClazz": "<class 'type'>",
+            "liveIdentity": "140258331079168",
+            "presentation": null
+          },
+          {
+            "name": "random",
+            "value": "<module 'random' from '/usr/local/lib/python3.8/random.py'>",
+            "lineNumber": -1,
+            "scope": "LOCAL_VARIABLE",
+            "liveClazz": "<class 'module'>",
+            "liveIdentity": "140259914813504",
+            "presentation": null
+          },
+          {
+            "name": "random_number",
+            "value": "49.90006897806545",
+            "lineNumber": -1,
+            "scope": "LOCAL_VARIABLE",
+            "liveClazz": "<class 'float'>",
+            "liveIdentity": "140259851605296",
+            "presentation": null
+          },
+          {
+            "name": "is_even",
+            "value": "False",
+            "lineNumber": -1,
+            "scope": "LOCAL_VARIABLE",
+            "liveClazz": "<class 'bool'>",
+            "liveIdentity": "140259935470944",
+            "presentation": null
+          }
+        ],
+        "sourceCode": "print(f\"{random_number} and is \" + (\"even\" if is_even else \"odd\"))"
+      },
+      {
+        "method": "trigger_add_breakpoint",
+        "source": "src/main.py:10",
+        "column": null,
+        "variables": [],
+        "sourceCode": "command.AddBreakpoint.AddBreakpoint.simple_breakpoint()"
+      },
+      {
+        "method": "execute_demos",
+        "source": "src/main.py:15",
+        "column": null,
+        "variables": [],
+        "sourceCode": "trigger_add_breakpoint()"
+      },
+      {
+        "method": "run",
+        "source": "/usr/local/lib/python3.8/threading.py:870",
+        "column": null,
+        "variables": [],
+        "sourceCode": "self._target(*self._args, **self._kwargs)"
+      },
+      {
+        "method": "_bootstrap_inner",
+        "source": "/usr/local/lib/python3.8/threading.py:932",
+        "column": null,
+        "variables": [],
+        "sourceCode": "self.run()"
+      },
+      {
+        "method": "_bootstrap",
+        "source": "/usr/local/lib/python3.8/threading.py:890",
+        "column": null,
+        "variables": [],
+        "sourceCode": "self._bootstrap_inner()"
+      }
+    ],
+    "causedBy": null,
+    "language": "PYTHON"
+  },
+  "eventType": "BREAKPOINT_HIT"
+}