From f964123a96a5c6c21478cb2667cf6a5fb49c3056 Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Fri, 28 Nov 2025 16:33:38 +0200 Subject: [PATCH 01/15] Add MCP conformance test coverage --- .github/workflows/build.yml | 15 + .gitignore | 3 + kotlin-sdk-test/build.gradle.kts | 37 ++ .../sdk/conformance/ConformanceClient.kt | 100 +++++ .../sdk/conformance/ConformanceServer.kt | 371 ++++++++++++++++++ .../kotlin/sdk/conformance/ConformanceTest.kt | 264 +++++++++++++ 6 files changed, 790 insertions(+) create mode 100644 kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt create mode 100644 kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt create mode 100644 kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9eed688a..a12c911f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,6 +28,11 @@ jobs: java-version: '21' distribution: 'temurin' + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Setup Gradle uses: gradle/actions/setup-gradle@v5 with: @@ -57,6 +62,16 @@ jobs: working-directory: ./samples/weather-stdio-server run: ./gradlew --no-daemon clean build -Pmcp.kotlin.overrideVersion=0.0.1-SNAPSHOT + - name: Run Conformance Tests + run: ./gradlew conformance + + - name: Upload Conformance Results + if: always() + uses: actions/upload-artifact@v5 + with: + name: conformance-results + path: kotlin-sdk-test/results/ + - name: Upload Reports if: ${{ !cancelled() }} uses: actions/upload-artifact@v5 diff --git a/.gitignore b/.gitignore index ea62a956..384748b7 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,6 @@ dist ### SWE agents ### .claude/ .junie/ + +### Conformance test results ### +kotlin-sdk-test/results/ diff --git a/kotlin-sdk-test/build.gradle.kts b/kotlin-sdk-test/build.gradle.kts index 252d0e21..caf88b23 100644 --- a/kotlin-sdk-test/build.gradle.kts +++ b/kotlin-sdk-test/build.gradle.kts @@ -1,3 +1,6 @@ +import org.gradle.api.tasks.testing.logging.TestExceptionFormat +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJvmCompilation + plugins { id("mcp.multiplatform") } @@ -31,3 +34,37 @@ kotlin { } } } + +tasks.register("conformance") { + group = "conformance" + description = "Run MCP conformance tests with detailed output" + + val jvmCompilation = kotlin.targets["jvm"].compilations["test"] as KotlinJvmCompilation + testClassesDirs = jvmCompilation.output.classesDirs + classpath = jvmCompilation.runtimeDependencyFiles + + useJUnitPlatform() + + filter { + includeTestsMatching("*ConformanceTest*") + } + + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showExceptions = true + showCauses = true + showStackTraces = true + exceptionFormat = TestExceptionFormat.FULL + } + + doFirst { + systemProperty("test.classpath", classpath.asPath) + + println("\n" + "=".repeat(60)) + println("MCP CONFORMANCE TESTS") + println("=".repeat(60)) + println("These tests validate compliance with the MCP specification.") + println("=".repeat(60) + "\n") + } +} diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt new file mode 100644 index 00000000..fd83068f --- /dev/null +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt @@ -0,0 +1,100 @@ +package io.modelcontextprotocol.kotlin.sdk.conformance + +import io.github.oshai.kotlinlogging.KotlinLogging +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.sse.SSE +import io.modelcontextprotocol.kotlin.sdk.client.Client +import io.modelcontextprotocol.kotlin.sdk.client.StreamableHttpClientTransport +import io.modelcontextprotocol.kotlin.sdk.shared.Transport +import io.modelcontextprotocol.kotlin.sdk.types.CallToolRequest +import io.modelcontextprotocol.kotlin.sdk.types.CallToolRequestParams +import io.modelcontextprotocol.kotlin.sdk.types.Implementation +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject + +private val logger = KotlinLogging.logger {} + +fun main(args: Array) { + require(args.isNotEmpty()) { + "Usage: ConformanceClient " + } + + val serverUrl = args.last() + logger.info { "Connecting to test server at: $serverUrl" } + + val httpClient = HttpClient(CIO) { + install(SSE) + } + val transport: Transport = StreamableHttpClientTransport(httpClient, serverUrl) + + val client = Client( + clientInfo = Implementation( + name = "kotlin-conformance-client", + version = "1.0.0" + ) + ) + + var exitCode = 0 + + runBlocking { + try { + client.connect(transport) + logger.info { "✅ Connected to server successfully" } + + try { + val tools = client.listTools() + logger.info { "Available tools: ${tools.tools.map { it.name }}" } + + val addNumbersTool = tools.tools.find { it.name == "add_numbers" } + if (addNumbersTool != null) { + logger.info { "Calling tool: add_numbers" } + val result = client.callTool( + CallToolRequest( + params = CallToolRequestParams( + name = "add_numbers", + arguments = buildJsonObject { + put("a", JsonPrimitive(5)) + put("b", JsonPrimitive(3)) + } + ) + ) + ) + logger.info { "Tool result: ${result.content}" } + } else if (tools.tools.isNotEmpty()) { + val toolName = tools.tools.first().name + logger.info { "Calling tool: $toolName" } + + val result = client.callTool( + CallToolRequest( + params = CallToolRequestParams( + name = toolName, + arguments = buildJsonObject { + put("input", JsonPrimitive("test")) + } + ) + ) + ) + logger.info { "Tool result: ${result.content}" } + } + } catch (e: Exception) { + logger.debug(e) { "Error during tool operations (may be expected for some scenarios)" } + } + + logger.info { "✅ Client operations completed successfully" } + } catch (e: Exception) { + logger.error(e) { "❌ Client failed" } + exitCode = 1 + } finally { + try { + transport.close() + } catch (e: Exception) { + logger.warn(e) { "Error closing transport" } + } + httpClient.close() + } + } + + kotlin.system.exitProcess(exitCode) +} diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt new file mode 100644 index 00000000..a383f5dc --- /dev/null +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt @@ -0,0 +1,371 @@ +package io.modelcontextprotocol.kotlin.sdk.conformance + +import io.github.oshai.kotlinlogging.KotlinLogging +import io.ktor.http.ContentType +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.cio.CIO +import io.ktor.server.engine.embeddedServer +import io.ktor.server.request.header +import io.ktor.server.request.receiveText +import io.ktor.server.response.header +import io.ktor.server.response.respond +import io.ktor.server.response.respondText +import io.ktor.server.response.respondTextWriter +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.server.routing.routing +import io.modelcontextprotocol.kotlin.sdk.server.Server +import io.modelcontextprotocol.kotlin.sdk.server.ServerOptions +import io.modelcontextprotocol.kotlin.sdk.shared.AbstractTransport +import io.modelcontextprotocol.kotlin.sdk.shared.TransportSendOptions +import io.modelcontextprotocol.kotlin.sdk.types.CallToolResult +import io.modelcontextprotocol.kotlin.sdk.types.GetPromptResult +import io.modelcontextprotocol.kotlin.sdk.types.Implementation +import io.modelcontextprotocol.kotlin.sdk.types.JSONRPCError +import io.modelcontextprotocol.kotlin.sdk.types.JSONRPCMessage +import io.modelcontextprotocol.kotlin.sdk.types.JSONRPCRequest +import io.modelcontextprotocol.kotlin.sdk.types.JSONRPCResponse +import io.modelcontextprotocol.kotlin.sdk.types.McpJson +import io.modelcontextprotocol.kotlin.sdk.types.PromptArgument +import io.modelcontextprotocol.kotlin.sdk.types.PromptMessage +import io.modelcontextprotocol.kotlin.sdk.types.RPCError +import io.modelcontextprotocol.kotlin.sdk.types.ReadResourceResult +import io.modelcontextprotocol.kotlin.sdk.types.RequestId +import io.modelcontextprotocol.kotlin.sdk.types.Role +import io.modelcontextprotocol.kotlin.sdk.types.ServerCapabilities +import io.modelcontextprotocol.kotlin.sdk.types.TextContent +import io.modelcontextprotocol.kotlin.sdk.types.TextResourceContents +import io.modelcontextprotocol.kotlin.sdk.types.ToolSchema +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.decodeFromJsonElement +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.put +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap + +private val logger = KotlinLogging.logger {} +private val serverTransports = ConcurrentHashMap() +private val jsonFormat = Json { ignoreUnknownKeys = true } + +private fun isInitializeRequest(json: JsonElement): Boolean = + json is JsonObject && json["method"]?.jsonPrimitive?.contentOrNull == "initialize" + +fun main(args: Array) { + val port = args.getOrNull(0)?.toIntOrNull() ?: 3000 + + logger.info { "Starting MCP Conformance Server on port $port" } + + embeddedServer(CIO, port = port, host = "127.0.0.1") { + routing { + get("/mcp") { + val sessionId = call.request.header("mcp-session-id") + if (sessionId == null) { + call.respond(HttpStatusCode.BadRequest, "Missing mcp-session-id header") + return@get + } + val transport = serverTransports[sessionId] + if (transport == null) { + call.respond(HttpStatusCode.BadRequest, "Invalid mcp-session-id") + return@get + } + transport.stream(call) + } + + post("/mcp") { + val sessionId = call.request.header("mcp-session-id") + val requestBody = call.receiveText() + + logger.debug { "Received request with sessionId: $sessionId" } + logger.trace { "Request body: $requestBody" } + + val jsonElement = try { + jsonFormat.parseToJsonElement(requestBody) + } catch (e: Exception) { + logger.error(e) { "Failed to parse request body as JSON" } + call.respond( + HttpStatusCode.BadRequest, + jsonFormat.encodeToString( + JsonObject.serializer(), + JsonObject( + mapOf( + "jsonrpc" to JsonPrimitive("2.0"), + "error" to JsonObject( + mapOf( + "code" to JsonPrimitive(-32700), + "message" to JsonPrimitive("Parse error: ${e.message}"), + ), + ), + "id" to JsonNull, + ), + ), + ), + ) + return@post + } + + if (sessionId != null && serverTransports.containsKey(sessionId)) { + logger.debug { "Using existing transport for session: $sessionId" } + val transport = serverTransports[sessionId]!! + transport.handleRequest(call, jsonElement) + } else { + if (isInitializeRequest(jsonElement)) { + val newSessionId = UUID.randomUUID().toString() + logger.info { "Creating new session with ID: $newSessionId" } + + val transport = HttpServerTransport(newSessionId) + serverTransports[newSessionId] = transport + + val mcpServer = createConformanceServer() + call.response.header("mcp-session-id", newSessionId) + + val sessionReady = CompletableDeferred() + Thread { + runBlocking { + try { + mcpServer.createSession(transport) + sessionReady.complete(Unit) + } catch (e: Exception) { + logger.error(e) { "Failed to create session" } + sessionReady.completeExceptionally(e) + } + } + }.start() + + runBlocking { + withTimeoutOrNull(2000) { + sessionReady.await() + } ?: logger.warn { "Session creation timed out, proceeding anyway" } + } + + transport.handleRequest(call, jsonElement) + } else { + logger.warn { "Invalid request: no session ID or not an initialization request" } + call.respond( + HttpStatusCode.BadRequest, + jsonFormat.encodeToString( + JsonObject.serializer(), + JsonObject( + mapOf( + "jsonrpc" to JsonPrimitive("2.0"), + "error" to JsonObject( + mapOf( + "code" to JsonPrimitive(-32000), + "message" to + JsonPrimitive("Bad Request: No valid session ID provided"), + ), + ), + "id" to JsonNull, + ), + ), + ), + ) + } + } + } + + delete("/mcp") { + val sessionId = call.request.header("mcp-session-id") + if (sessionId != null && serverTransports.containsKey(sessionId)) { + logger.info { "Terminating session: $sessionId" } + val transport = serverTransports[sessionId]!! + serverTransports.remove(sessionId) + runBlocking { + transport.close() + } + call.respond(HttpStatusCode.OK) + } else { + logger.warn { "Invalid session termination request: $sessionId" } + call.respond(HttpStatusCode.BadRequest, "Invalid or missing session ID") + } + } + } + }.start(wait = true) +} + +private fun createConformanceServer(): Server { + val server = Server( + Implementation( + name = "kotlin-conformance-server", + version = "1.0.0" + ), + ServerOptions( + capabilities = ServerCapabilities( + tools = ServerCapabilities.Tools(listChanged = true), + resources = ServerCapabilities.Resources(subscribe = true, listChanged = true), + prompts = ServerCapabilities.Prompts(listChanged = true) + ) + ) + ) + + server.addTool( + name = "test_tool", + description = "A test tool for conformance testing", + inputSchema = ToolSchema( + properties = buildJsonObject { + put("input", buildJsonObject { + put("type", "string") + put("description", "Test input parameter") + }) + }, + required = listOf("input") + ) + ) { request -> + val input = (request.params.arguments?.get("input") as? JsonPrimitive)?.content ?: "no input" + CallToolResult( + content = listOf(TextContent("Tool executed with input: $input")) + ) + } + + server.addResource( + uri = "test://resource", + name = "Test Resource", + description = "A test resource for conformance testing", + mimeType = "text/plain" + ) { request -> + ReadResourceResult( + contents = listOf( + TextResourceContents("Test resource content", request.params.uri, "text/plain") + ) + ) + } + + server.addPrompt( + name = "test_prompt", + description = "A test prompt for conformance testing", + arguments = listOf( + PromptArgument( + name = "arg", + description = "Test argument", + required = false + ) + ) + ) { + GetPromptResult( + messages = listOf( + PromptMessage( + role = Role.User, + content = TextContent("Test prompt content") + ) + ), + description = "Test prompt description" + ) + } + + return server +} + +private class HttpServerTransport(private val sessionId: String) : AbstractTransport() { + private val logger = KotlinLogging.logger {} + private val pendingResponses = ConcurrentHashMap>() + private val messageQueue = Channel(Channel.UNLIMITED) + + suspend fun stream(call: ApplicationCall) { + logger.debug { "Starting SSE stream for session $sessionId" } + call.response.apply { + header("Cache-Control", "no-cache") + header("Connection", "keep-alive") + } + call.respondTextWriter(ContentType.Text.EventStream) { + try { + while (true) { + val msg = messageQueue.receiveCatching().getOrNull() ?: break + write("event: message\ndata: ${McpJson.encodeToString(msg)}\n\n") + flush() + } + } catch (e: Exception) { + logger.warn(e) { "SSE stream terminated for session $sessionId" } + } finally { + logger.debug { "SSE stream closed for session $sessionId" } + } + } + } + + suspend fun handleRequest(call: ApplicationCall, requestBody: JsonElement) { + try { + val message = McpJson.decodeFromJsonElement(requestBody) + logger.debug { "Handling ${message::class.simpleName}: $requestBody" } + + when (message) { + is JSONRPCRequest -> { + val id = message.id.toString() + val responseDeferred = CompletableDeferred() + pendingResponses[id] = responseDeferred + + _onMessage.invoke(message) + + val response = withTimeoutOrNull(10_000) { responseDeferred.await() } + if (response != null) { + call.respondText(McpJson.encodeToString(response), ContentType.Application.Json) + } else { + logger.warn { "Timeout for request $id" } + call.respondText( + McpJson.encodeToString( + JSONRPCError( + message.id, + RPCError(RPCError.ErrorCode.REQUEST_TIMEOUT, "Request timed out") + ) + ), + ContentType.Application.Json + ) + } + } + + else -> call.respond(HttpStatusCode.Accepted) + } + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + logger.error(e) { "Error handling request" } + if (!call.response.isCommitted) { + call.respondText( + McpJson.encodeToString( + JSONRPCError( + RequestId(0), + RPCError(RPCError.ErrorCode.INTERNAL_ERROR, "Internal error: ${e.message}") + ) + ), + ContentType.Application.Json, + HttpStatusCode.InternalServerError + ) + } + } + } + + override suspend fun start() { + logger.debug { "Started transport for session $sessionId" } + } + + override suspend fun send(message: JSONRPCMessage, options: TransportSendOptions?) { + when (message) { + is JSONRPCResponse -> { + val id = message.id.toString() + pendingResponses.remove(id)?.complete(message) ?: run { + logger.warn { "No pending response for ID $id, queueing" } + messageQueue.send(message) + } + } + + else -> messageQueue.send(message) + } + } + + override suspend fun close() { + logger.debug { "Closing transport for session $sessionId" } + messageQueue.close() + pendingResponses.clear() + _onClose.invoke() + } +} diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt new file mode 100644 index 00000000..e9ec80c0 --- /dev/null +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt @@ -0,0 +1,264 @@ +package io.modelcontextprotocol.kotlin.sdk.conformance + +import io.github.oshai.kotlinlogging.KotlinLogging +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.DynamicTest +import org.junit.jupiter.api.TestFactory +import org.junit.jupiter.api.TestInstance +import java.io.BufferedReader +import java.io.InputStreamReader +import java.lang.management.ManagementFactory +import java.net.HttpURLConnection +import java.net.ServerSocket +import java.net.URI +import java.util.concurrent.TimeUnit + +private val logger = KotlinLogging.logger {} + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class ConformanceTest { + + private var serverProcess: Process? = null + private var serverPort: Int = 0 + private val serverErrorOutput = StringBuilder() + + companion object { + private val SERVER_SCENARIOS = listOf( + "server-initialize", + "tools-list", + "tools-call-simple-text", + "resources-list", + "prompts-list", + // TODO: Fix + // The following scenarios are failing (likely due to us not meeting the latest specification): + // - resources-read-text + // - prompts-get-simple + ) + + private val CLIENT_SCENARIOS = listOf( + "initialize", + // TODO: Fix + // The following scenarios are failing (likely due to us not meeting the latest specification): + // "tools-call", + ) + + private const val DEFAULT_TEST_TIMEOUT_SECONDS = 30L + private const val DEFAULT_SERVER_STARTUP_TIMEOUT_SECONDS = 10 + + private fun findFreePort(): Int { + return ServerSocket(0).use { it.localPort } + } + + private fun getRuntimeClasspath(): String { + return ManagementFactory.getRuntimeMXBean().classPath + } + + private fun getTestClasspath(): String { + return System.getProperty("test.classpath") ?: getRuntimeClasspath() + } + + private fun waitForServerReady( + url: String, + timeoutSeconds: Int = DEFAULT_SERVER_STARTUP_TIMEOUT_SECONDS, + ): Boolean { + val deadline = System.currentTimeMillis() + (timeoutSeconds * 1000) + var lastError: Exception? = null + var backoffMs = 50L + + while (System.currentTimeMillis() < deadline) { + try { + val connection = URI(url).toURL().openConnection() as HttpURLConnection + connection.requestMethod = "GET" + connection.connectTimeout = 500 + connection.readTimeout = 500 + connection.connect() + + val responseCode = connection.responseCode + connection.disconnect() + logger.debug { "Server responded with code: $responseCode" } + return true + } catch (e: Exception) { + lastError = e + Thread.sleep(backoffMs) + backoffMs = (backoffMs * 1.5).toLong().coerceAtMost(500) + } + } + + logger.error { "Server did not start within $timeoutSeconds seconds. Last error: ${lastError?.message}" } + return false + } + } + + @BeforeAll + fun startServer() { + serverPort = findFreePort() + val serverUrl = "http://127.0.0.1:$serverPort/mcp" + + logger.info { "Starting conformance test server on port $serverPort" } + + val processBuilder = ProcessBuilder( + "java", + "-cp", getRuntimeClasspath(), + "io.modelcontextprotocol.kotlin.sdk.conformance.ConformanceServerKt", + serverPort.toString() + ) + + serverProcess = processBuilder.start() + + // capture stderr in the background + Thread { + try { + BufferedReader(InputStreamReader(serverProcess!!.errorStream)).use { reader -> + reader.lineSequence().forEach { line -> + serverErrorOutput.appendLine(line) + logger.debug { "Server stderr: $line" } + } + } + } catch (e: Exception) { + logger.trace(e) { "Error reading server stderr" } + } + }.start() + + logger.info { "Waiting for server to start..." } + val serverReady = waitForServerReady(serverUrl) + + if (!serverReady) { + val errorInfo = if (serverErrorOutput.isNotEmpty()) { + "\n\nServer error output:\n${serverErrorOutput}" + } else { + "" + } + serverProcess?.destroyForcibly() + throw IllegalStateException( + "Server failed to start within $DEFAULT_SERVER_STARTUP_TIMEOUT_SECONDS seconds. " + + "Check if port $serverPort is available.$errorInfo" + ) + } + + logger.info { "Server started successfully at $serverUrl" } + } + + @AfterAll + fun stopServer() { + if (serverProcess == null) { + logger.debug { "No server process to stop" } + return + } + + logger.info { "Stopping conformance test server (PID: ${serverProcess?.pid()})" } + + try { + serverProcess?.destroy() + val terminated = serverProcess?.waitFor(5, TimeUnit.SECONDS) ?: false + + if (!terminated) { + logger.warn { "Server did not terminate gracefully, forcing shutdown..." } + serverProcess?.destroyForcibly() + serverProcess?.waitFor(2, TimeUnit.SECONDS) ?: false + } else { + logger.info { "Server stopped gracefully" } + } + } catch (e: Exception) { + logger.error(e) { "Error stopping server process" } + } finally { + serverProcess = null + } + } + + @TestFactory + fun `MCP Server Conformance Tests`(): List { + val serverUrl = "http://127.0.0.1:$serverPort/mcp" + + return SERVER_SCENARIOS.map { scenario -> + DynamicTest.dynamicTest("Server: $scenario") { + runServerConformanceTest(scenario, serverUrl) + } + } + } + + @TestFactory + fun `MCP Client Conformance Tests`(): List { + return CLIENT_SCENARIOS.map { scenario -> + DynamicTest.dynamicTest("Client: $scenario") { + runClientConformanceTest(scenario) + } + } + } + + private fun runServerConformanceTest(scenario: String, serverUrl: String) { + logger.info { "Running server conformance test: $scenario" } + + val timeoutSeconds = System.getenv("CONFORMANCE_TEST_TIMEOUT_SECONDS")?.toLongOrNull() + ?: DEFAULT_TEST_TIMEOUT_SECONDS + + val process = ProcessBuilder( + "npx", + "@modelcontextprotocol/conformance", + "server", + "--url", serverUrl, + "--scenario", scenario + ).apply { + inheritIO() + }.start() + + val completed = process.waitFor(timeoutSeconds, TimeUnit.SECONDS) + + if (!completed) { + logger.error { "Server conformance test '$scenario' timed out after $timeoutSeconds seconds" } + process.destroyForcibly() + throw AssertionError("❌ Server conformance test '$scenario' timed out after $timeoutSeconds seconds") + } + + val exitCode = process.exitValue() + + if (exitCode != 0) { + logger.error { "Server conformance test '$scenario' failed with exit code: $exitCode" } + throw AssertionError("❌ Server conformance test '$scenario' failed (exit code: $exitCode). Check test output above for details.") + } + + logger.info { "✅ Server conformance test '$scenario' passed!" } + } + + private fun runClientConformanceTest(scenario: String) { + logger.info { "Running client conformance test: $scenario" } + + val timeoutSeconds = System.getenv("CONFORMANCE_TEST_TIMEOUT_SECONDS")?.toLongOrNull() + ?: DEFAULT_TEST_TIMEOUT_SECONDS + + val testClasspath = getTestClasspath() + + val clientCommand = listOf( + "java", + "-cp", testClasspath, + "io.modelcontextprotocol.kotlin.sdk.conformance.ConformanceClientKt" + ) + + val process = ProcessBuilder( + "npx", + "@modelcontextprotocol/conformance", + "client", + "--command", clientCommand.joinToString(" "), + "--scenario", scenario + ).apply { + inheritIO() + }.start() + + val completed = process.waitFor(timeoutSeconds, TimeUnit.SECONDS) + + if (!completed) { + logger.error { "Client conformance test '$scenario' timed out after $timeoutSeconds seconds" } + process.destroyForcibly() + throw AssertionError("❌ Client conformance test '$scenario' timed out after $timeoutSeconds seconds") + } + + val exitCode = process.exitValue() + + if (exitCode != 0) { + logger.error { "Client conformance test '$scenario' failed with exit code: $exitCode" } + throw AssertionError("❌ Client conformance test '$scenario' failed (exit code: $exitCode). Check test output above for details.") + } + + logger.info { "✅ Client conformance test '$scenario' passed!" } + } +} From 59b64007f9904607f81c499d25b9bf5ba64b2e9d Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Fri, 28 Nov 2025 16:52:02 +0200 Subject: [PATCH 02/15] fixup! Add MCP conformance test coverage --- .../kotlin/sdk/conformance/ConformanceTest.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt index e9ec80c0..09f0372d 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt @@ -31,7 +31,6 @@ class ConformanceTest { "resources-list", "prompts-list", // TODO: Fix - // The following scenarios are failing (likely due to us not meeting the latest specification): // - resources-read-text // - prompts-get-simple ) @@ -39,7 +38,6 @@ class ConformanceTest { private val CLIENT_SCENARIOS = listOf( "initialize", // TODO: Fix - // The following scenarios are failing (likely due to us not meeting the latest specification): // "tools-call", ) From 65b244f02e1d0098023cd5ed709984e00ee2cac4 Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Fri, 28 Nov 2025 17:03:33 +0200 Subject: [PATCH 03/15] fixup! Add MCP conformance test coverage --- .../kotlin/sdk/conformance/ConformanceTest.kt | 130 +++++++++--------- 1 file changed, 63 insertions(+), 67 deletions(-) diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt index 09f0372d..64d60dd7 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt @@ -13,6 +13,7 @@ import java.net.HttpURLConnection import java.net.ServerSocket import java.net.URI import java.util.concurrent.TimeUnit +import kotlin.properties.Delegates private val logger = KotlinLogging.logger {} @@ -20,8 +21,8 @@ private val logger = KotlinLogging.logger {} class ConformanceTest { private var serverProcess: Process? = null - private var serverPort: Int = 0 - private val serverErrorOutput = StringBuilder() + private var serverPort: Int by Delegates.notNull() + private val serverErrorOutput = StringBuffer() companion object { private val SERVER_SCENARIOS = listOf( @@ -43,6 +44,12 @@ class ConformanceTest { private const val DEFAULT_TEST_TIMEOUT_SECONDS = 30L private const val DEFAULT_SERVER_STARTUP_TIMEOUT_SECONDS = 10 + private const val INITIAL_BACKOFF_MS = 50L + private const val MAX_BACKOFF_MS = 500L + private const val BACKOFF_MULTIPLIER = 1.5 + private const val CONNECTION_TIMEOUT_MS = 500 + private const val GRACEFUL_SHUTDOWN_SECONDS = 5L + private const val FORCE_SHUTDOWN_SECONDS = 2L private fun findFreePort(): Int { return ServerSocket(0).use { it.localPort } @@ -62,14 +69,14 @@ class ConformanceTest { ): Boolean { val deadline = System.currentTimeMillis() + (timeoutSeconds * 1000) var lastError: Exception? = null - var backoffMs = 50L + var backoffMs = INITIAL_BACKOFF_MS while (System.currentTimeMillis() < deadline) { try { val connection = URI(url).toURL().openConnection() as HttpURLConnection connection.requestMethod = "GET" - connection.connectTimeout = 500 - connection.readTimeout = 500 + connection.connectTimeout = CONNECTION_TIMEOUT_MS + connection.readTimeout = CONNECTION_TIMEOUT_MS connection.connect() val responseCode = connection.responseCode @@ -79,7 +86,7 @@ class ConformanceTest { } catch (e: Exception) { lastError = e Thread.sleep(backoffMs) - backoffMs = (backoffMs * 1.5).toLong().coerceAtMost(500) + backoffMs = (backoffMs * BACKOFF_MULTIPLIER).toLong().coerceAtMost(MAX_BACKOFF_MS) } } @@ -102,12 +109,13 @@ class ConformanceTest { serverPort.toString() ) - serverProcess = processBuilder.start() + val process = processBuilder.start() + serverProcess = process // capture stderr in the background Thread { try { - BufferedReader(InputStreamReader(serverProcess!!.errorStream)).use { reader -> + BufferedReader(InputStreamReader(process.errorStream)).use { reader -> reader.lineSequence().forEach { line -> serverErrorOutput.appendLine(line) logger.debug { "Server stderr: $line" } @@ -116,6 +124,9 @@ class ConformanceTest { } catch (e: Exception) { logger.trace(e) { "Error reading server stderr" } } + }.apply { + name = "server-stderr-reader" + isDaemon = true }.start() logger.info { "Waiting for server to start..." } @@ -139,29 +150,26 @@ class ConformanceTest { @AfterAll fun stopServer() { - if (serverProcess == null) { - logger.debug { "No server process to stop" } - return - } - - logger.info { "Stopping conformance test server (PID: ${serverProcess?.pid()})" } - - try { - serverProcess?.destroy() - val terminated = serverProcess?.waitFor(5, TimeUnit.SECONDS) ?: false + serverProcess?.also { process -> + logger.info { "Stopping conformance test server (PID: ${process.pid()})" } - if (!terminated) { - logger.warn { "Server did not terminate gracefully, forcing shutdown..." } - serverProcess?.destroyForcibly() - serverProcess?.waitFor(2, TimeUnit.SECONDS) ?: false - } else { - logger.info { "Server stopped gracefully" } + try { + process.destroy() + val terminated = process.waitFor(GRACEFUL_SHUTDOWN_SECONDS, TimeUnit.SECONDS) + + if (!terminated) { + logger.warn { "Server did not terminate gracefully, forcing shutdown..." } + process.destroyForcibly() + process.waitFor(FORCE_SHUTDOWN_SECONDS, TimeUnit.SECONDS) + } else { + logger.info { "Server stopped gracefully" } + } + } catch (e: Exception) { + logger.error(e) { "Error stopping server process" } + } finally { + serverProcess = null } - } catch (e: Exception) { - logger.error(e) { "Error stopping server process" } - } finally { - serverProcess = null - } + } ?: logger.debug { "No server process to stop" } } @TestFactory @@ -185,12 +193,7 @@ class ConformanceTest { } private fun runServerConformanceTest(scenario: String, serverUrl: String) { - logger.info { "Running server conformance test: $scenario" } - - val timeoutSeconds = System.getenv("CONFORMANCE_TEST_TIMEOUT_SECONDS")?.toLongOrNull() - ?: DEFAULT_TEST_TIMEOUT_SECONDS - - val process = ProcessBuilder( + val processBuilder = ProcessBuilder( "npx", "@modelcontextprotocol/conformance", "server", @@ -198,32 +201,12 @@ class ConformanceTest { "--scenario", scenario ).apply { inheritIO() - }.start() - - val completed = process.waitFor(timeoutSeconds, TimeUnit.SECONDS) - - if (!completed) { - logger.error { "Server conformance test '$scenario' timed out after $timeoutSeconds seconds" } - process.destroyForcibly() - throw AssertionError("❌ Server conformance test '$scenario' timed out after $timeoutSeconds seconds") } - val exitCode = process.exitValue() - - if (exitCode != 0) { - logger.error { "Server conformance test '$scenario' failed with exit code: $exitCode" } - throw AssertionError("❌ Server conformance test '$scenario' failed (exit code: $exitCode). Check test output above for details.") - } - - logger.info { "✅ Server conformance test '$scenario' passed!" } + runConformanceTest("server", scenario, processBuilder) } private fun runClientConformanceTest(scenario: String) { - logger.info { "Running client conformance test: $scenario" } - - val timeoutSeconds = System.getenv("CONFORMANCE_TEST_TIMEOUT_SECONDS")?.toLongOrNull() - ?: DEFAULT_TEST_TIMEOUT_SECONDS - val testClasspath = getTestClasspath() val clientCommand = listOf( @@ -232,7 +215,7 @@ class ConformanceTest { "io.modelcontextprotocol.kotlin.sdk.conformance.ConformanceClientKt" ) - val process = ProcessBuilder( + val processBuilder = ProcessBuilder( "npx", "@modelcontextprotocol/conformance", "client", @@ -240,23 +223,36 @@ class ConformanceTest { "--scenario", scenario ).apply { inheritIO() - }.start() + } + runConformanceTest("client", scenario, processBuilder) + } + + private fun runConformanceTest( + type: String, + scenario: String, + processBuilder: ProcessBuilder + ) { + logger.info { "Running $type conformance test: $scenario" } + + val timeoutSeconds = System.getenv("CONFORMANCE_TEST_TIMEOUT_SECONDS")?.toLongOrNull() + ?: DEFAULT_TEST_TIMEOUT_SECONDS + + val process = processBuilder.start() val completed = process.waitFor(timeoutSeconds, TimeUnit.SECONDS) if (!completed) { - logger.error { "Client conformance test '$scenario' timed out after $timeoutSeconds seconds" } + logger.error { "${type.replaceFirstChar { it.uppercase() }} conformance test '$scenario' timed out after $timeoutSeconds seconds" } process.destroyForcibly() - throw AssertionError("❌ Client conformance test '$scenario' timed out after $timeoutSeconds seconds") + throw AssertionError("❌ ${type.replaceFirstChar { it.uppercase() }} conformance test '$scenario' timed out after $timeoutSeconds seconds") } - val exitCode = process.exitValue() - - if (exitCode != 0) { - logger.error { "Client conformance test '$scenario' failed with exit code: $exitCode" } - throw AssertionError("❌ Client conformance test '$scenario' failed (exit code: $exitCode). Check test output above for details.") + when (val exitCode = process.exitValue()) { + 0 -> logger.info { "✅ ${type.replaceFirstChar { it.uppercase() }} conformance test '$scenario' passed!" } + else -> { + logger.error { "${type.replaceFirstChar { it.uppercase() }} conformance test '$scenario' failed with exit code: $exitCode" } + throw AssertionError("❌ ${type.replaceFirstChar { it.uppercase() }} conformance test '$scenario' failed (exit code: $exitCode). Check test output above for details.") + } } - - logger.info { "✅ Client conformance test '$scenario' passed!" } } } From c8a4a8641f85e7d99747e2da3a8e06f85fab023c Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Fri, 28 Nov 2025 17:11:38 +0200 Subject: [PATCH 04/15] fixup! Add MCP conformance test coverage --- .../sdk/conformance/ConformanceServer.kt | 134 +++++++++--------- 1 file changed, 66 insertions(+), 68 deletions(-) diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt index a383f5dc..ea2b34d9 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt @@ -40,7 +40,10 @@ import io.modelcontextprotocol.kotlin.sdk.types.TextResourceContents import io.modelcontextprotocol.kotlin.sdk.types.ToolSchema import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeoutOrNull import kotlinx.serialization.json.Json @@ -60,6 +63,10 @@ private val logger = KotlinLogging.logger {} private val serverTransports = ConcurrentHashMap() private val jsonFormat = Json { ignoreUnknownKeys = true } +private const val SESSION_CREATION_TIMEOUT_MS = 2000L +private const val REQUEST_TIMEOUT_MS = 10_000L +private const val MESSAGE_QUEUE_CAPACITY = 256 + private fun isInitializeRequest(json: JsonElement): Boolean = json is JsonObject && json["method"]?.jsonPrimitive?.contentOrNull == "initialize" @@ -72,15 +79,15 @@ fun main(args: Array) { routing { get("/mcp") { val sessionId = call.request.header("mcp-session-id") - if (sessionId == null) { - call.respond(HttpStatusCode.BadRequest, "Missing mcp-session-id header") - return@get - } + ?: run { + call.respond(HttpStatusCode.BadRequest, "Missing mcp-session-id header") + return@get + } val transport = serverTransports[sessionId] - if (transport == null) { - call.respond(HttpStatusCode.BadRequest, "Invalid mcp-session-id") - return@get - } + ?: run { + call.respond(HttpStatusCode.BadRequest, "Invalid mcp-session-id") + return@get + } transport.stream(call) } @@ -99,78 +106,65 @@ fun main(args: Array) { HttpStatusCode.BadRequest, jsonFormat.encodeToString( JsonObject.serializer(), - JsonObject( - mapOf( - "jsonrpc" to JsonPrimitive("2.0"), - "error" to JsonObject( - mapOf( - "code" to JsonPrimitive(-32700), - "message" to JsonPrimitive("Parse error: ${e.message}"), - ), - ), - "id" to JsonNull, - ), - ), - ), + buildJsonObject { + put("jsonrpc", "2.0") + put("error", buildJsonObject { + put("code", -32700) + put("message", "Parse error: ${e.message}") + }) + put("id", JsonNull) + } + ) ) return@post } - if (sessionId != null && serverTransports.containsKey(sessionId)) { + val transport = sessionId?.let { serverTransports[it] } + if (transport != null) { logger.debug { "Using existing transport for session: $sessionId" } - val transport = serverTransports[sessionId]!! transport.handleRequest(call, jsonElement) } else { if (isInitializeRequest(jsonElement)) { val newSessionId = UUID.randomUUID().toString() logger.info { "Creating new session with ID: $newSessionId" } - val transport = HttpServerTransport(newSessionId) - serverTransports[newSessionId] = transport + val newTransport = HttpServerTransport(newSessionId) + serverTransports[newSessionId] = newTransport val mcpServer = createConformanceServer() call.response.header("mcp-session-id", newSessionId) val sessionReady = CompletableDeferred() - Thread { - runBlocking { - try { - mcpServer.createSession(transport) - sessionReady.complete(Unit) - } catch (e: Exception) { - logger.error(e) { "Failed to create session" } - sessionReady.completeExceptionally(e) - } + CoroutineScope(Dispatchers.IO).launch { + try { + mcpServer.createSession(newTransport) + sessionReady.complete(Unit) + } catch (e: Exception) { + logger.error(e) { "Failed to create session" } + sessionReady.completeExceptionally(e) } - }.start() - - runBlocking { - withTimeoutOrNull(2000) { - sessionReady.await() - } ?: logger.warn { "Session creation timed out, proceeding anyway" } } - transport.handleRequest(call, jsonElement) + withTimeoutOrNull(SESSION_CREATION_TIMEOUT_MS) { + sessionReady.await() + } ?: logger.warn { "Session creation timed out, proceeding anyway" } + + newTransport.handleRequest(call, jsonElement) } else { logger.warn { "Invalid request: no session ID or not an initialization request" } call.respond( HttpStatusCode.BadRequest, jsonFormat.encodeToString( JsonObject.serializer(), - JsonObject( - mapOf( - "jsonrpc" to JsonPrimitive("2.0"), - "error" to JsonObject( - mapOf( - "code" to JsonPrimitive(-32000), - "message" to - JsonPrimitive("Bad Request: No valid session ID provided"), - ), - ), - "id" to JsonNull, - ), - ), - ), + buildJsonObject { + put("jsonrpc", "2.0") + put("error", buildJsonObject { + put("code", -32000) + put("message", "Bad Request: No valid session ID provided") + }) + put("id", JsonNull) + } + ) ) } } @@ -178,13 +172,11 @@ fun main(args: Array) { delete("/mcp") { val sessionId = call.request.header("mcp-session-id") - if (sessionId != null && serverTransports.containsKey(sessionId)) { + val transport = sessionId?.let { serverTransports[it] } + if (transport != null) { logger.info { "Terminating session: $sessionId" } - val transport = serverTransports[sessionId]!! serverTransports.remove(sessionId) - runBlocking { - transport.close() - } + transport.close() call.respond(HttpStatusCode.OK) } else { logger.warn { "Invalid session termination request: $sessionId" } @@ -270,7 +262,7 @@ private fun createConformanceServer(): Server { private class HttpServerTransport(private val sessionId: String) : AbstractTransport() { private val logger = KotlinLogging.logger {} private val pendingResponses = ConcurrentHashMap>() - private val messageQueue = Channel(Channel.UNLIMITED) + private val messageQueue = Channel(MESSAGE_QUEUE_CAPACITY) suspend fun stream(call: ApplicationCall) { logger.debug { "Starting SSE stream for session $sessionId" } @@ -300,17 +292,20 @@ private class HttpServerTransport(private val sessionId: String) : AbstractTrans when (message) { is JSONRPCRequest -> { - val id = message.id.toString() + val idKey = when (val id = message.id) { + is RequestId.NumberId -> id.value.toString() + is RequestId.StringId -> id.value + } val responseDeferred = CompletableDeferred() - pendingResponses[id] = responseDeferred + pendingResponses[idKey] = responseDeferred _onMessage.invoke(message) - val response = withTimeoutOrNull(10_000) { responseDeferred.await() } + val response = withTimeoutOrNull(REQUEST_TIMEOUT_MS) { responseDeferred.await() } if (response != null) { call.respondText(McpJson.encodeToString(response), ContentType.Application.Json) } else { - logger.warn { "Timeout for request $id" } + logger.warn { "Timeout for request $idKey" } call.respondText( McpJson.encodeToString( JSONRPCError( @@ -351,9 +346,12 @@ private class HttpServerTransport(private val sessionId: String) : AbstractTrans override suspend fun send(message: JSONRPCMessage, options: TransportSendOptions?) { when (message) { is JSONRPCResponse -> { - val id = message.id.toString() - pendingResponses.remove(id)?.complete(message) ?: run { - logger.warn { "No pending response for ID $id, queueing" } + val idKey = when (val id = message.id) { + is RequestId.NumberId -> id.value.toString() + is RequestId.StringId -> id.value + } + pendingResponses.remove(idKey)?.complete(message) ?: run { + logger.warn { "No pending response for ID $idKey, queueing" } messageQueue.send(message) } } From f927aa450c1e03c8b92685ea874db2b526176ce9 Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Fri, 28 Nov 2025 17:12:17 +0200 Subject: [PATCH 05/15] fixup! Add MCP conformance test coverage --- .../sdk/conformance/ConformanceClient.kt | 16 ++-- .../sdk/conformance/ConformanceServer.kt | 86 ++++++++++--------- .../kotlin/sdk/conformance/ConformanceTest.kt | 77 ++++++++++------- 3 files changed, 99 insertions(+), 80 deletions(-) diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt index fd83068f..ef9b5267 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt @@ -32,8 +32,8 @@ fun main(args: Array) { val client = Client( clientInfo = Implementation( name = "kotlin-conformance-client", - version = "1.0.0" - ) + version = "1.0.0", + ), ) var exitCode = 0 @@ -57,9 +57,9 @@ fun main(args: Array) { arguments = buildJsonObject { put("a", JsonPrimitive(5)) put("b", JsonPrimitive(3)) - } - ) - ) + }, + ), + ), ) logger.info { "Tool result: ${result.content}" } } else if (tools.tools.isNotEmpty()) { @@ -72,9 +72,9 @@ fun main(args: Array) { name = toolName, arguments = buildJsonObject { put("input", JsonPrimitive("test")) - } - ) - ) + }, + ), + ), ) logger.info { "Tool result: ${result.content}" } } diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt index ea2b34d9..419be6c1 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt @@ -44,7 +44,6 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeoutOrNull import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonElement @@ -108,13 +107,16 @@ fun main(args: Array) { JsonObject.serializer(), buildJsonObject { put("jsonrpc", "2.0") - put("error", buildJsonObject { - put("code", -32700) - put("message", "Parse error: ${e.message}") - }) + put( + "error", + buildJsonObject { + put("code", -32700) + put("message", "Parse error: ${e.message}") + }, + ) put("id", JsonNull) - } - ) + }, + ), ) return@post } @@ -158,13 +160,16 @@ fun main(args: Array) { JsonObject.serializer(), buildJsonObject { put("jsonrpc", "2.0") - put("error", buildJsonObject { - put("code", -32000) - put("message", "Bad Request: No valid session ID provided") - }) + put( + "error", + buildJsonObject { + put("code", -32000) + put("message", "Bad Request: No valid session ID provided") + }, + ) put("id", JsonNull) - } - ) + }, + ), ) } } @@ -191,15 +196,15 @@ private fun createConformanceServer(): Server { val server = Server( Implementation( name = "kotlin-conformance-server", - version = "1.0.0" + version = "1.0.0", ), ServerOptions( capabilities = ServerCapabilities( tools = ServerCapabilities.Tools(listChanged = true), resources = ServerCapabilities.Resources(subscribe = true, listChanged = true), - prompts = ServerCapabilities.Prompts(listChanged = true) - ) - ) + prompts = ServerCapabilities.Prompts(listChanged = true), + ), + ), ) server.addTool( @@ -207,17 +212,20 @@ private fun createConformanceServer(): Server { description = "A test tool for conformance testing", inputSchema = ToolSchema( properties = buildJsonObject { - put("input", buildJsonObject { - put("type", "string") - put("description", "Test input parameter") - }) + put( + "input", + buildJsonObject { + put("type", "string") + put("description", "Test input parameter") + }, + ) }, - required = listOf("input") - ) + required = listOf("input"), + ), ) { request -> val input = (request.params.arguments?.get("input") as? JsonPrimitive)?.content ?: "no input" CallToolResult( - content = listOf(TextContent("Tool executed with input: $input")) + content = listOf(TextContent("Tool executed with input: $input")), ) } @@ -225,12 +233,12 @@ private fun createConformanceServer(): Server { uri = "test://resource", name = "Test Resource", description = "A test resource for conformance testing", - mimeType = "text/plain" + mimeType = "text/plain", ) { request -> ReadResourceResult( contents = listOf( - TextResourceContents("Test resource content", request.params.uri, "text/plain") - ) + TextResourceContents("Test resource content", request.params.uri, "text/plain"), + ), ) } @@ -241,18 +249,18 @@ private fun createConformanceServer(): Server { PromptArgument( name = "arg", description = "Test argument", - required = false - ) - ) + required = false, + ), + ), ) { GetPromptResult( messages = listOf( PromptMessage( role = Role.User, - content = TextContent("Test prompt content") - ) + content = TextContent("Test prompt content"), + ), ), - description = "Test prompt description" + description = "Test prompt description", ) } @@ -310,10 +318,10 @@ private class HttpServerTransport(private val sessionId: String) : AbstractTrans McpJson.encodeToString( JSONRPCError( message.id, - RPCError(RPCError.ErrorCode.REQUEST_TIMEOUT, "Request timed out") - ) + RPCError(RPCError.ErrorCode.REQUEST_TIMEOUT, "Request timed out"), + ), ), - ContentType.Application.Json + ContentType.Application.Json, ) } } @@ -329,11 +337,11 @@ private class HttpServerTransport(private val sessionId: String) : AbstractTrans McpJson.encodeToString( JSONRPCError( RequestId(0), - RPCError(RPCError.ErrorCode.INTERNAL_ERROR, "Internal error: ${e.message}") - ) + RPCError(RPCError.ErrorCode.INTERNAL_ERROR, "Internal error: ${e.message}"), + ), ), ContentType.Application.Json, - HttpStatusCode.InternalServerError + HttpStatusCode.InternalServerError, ) } } diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt index 64d60dd7..a4901d31 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt @@ -51,17 +51,11 @@ class ConformanceTest { private const val GRACEFUL_SHUTDOWN_SECONDS = 5L private const val FORCE_SHUTDOWN_SECONDS = 2L - private fun findFreePort(): Int { - return ServerSocket(0).use { it.localPort } - } + private fun findFreePort(): Int = ServerSocket(0).use { it.localPort } - private fun getRuntimeClasspath(): String { - return ManagementFactory.getRuntimeMXBean().classPath - } + private fun getRuntimeClasspath(): String = ManagementFactory.getRuntimeMXBean().classPath - private fun getTestClasspath(): String { - return System.getProperty("test.classpath") ?: getRuntimeClasspath() - } + private fun getTestClasspath(): String = System.getProperty("test.classpath") ?: getRuntimeClasspath() private fun waitForServerReady( url: String, @@ -104,9 +98,10 @@ class ConformanceTest { val processBuilder = ProcessBuilder( "java", - "-cp", getRuntimeClasspath(), + "-cp", + getRuntimeClasspath(), "io.modelcontextprotocol.kotlin.sdk.conformance.ConformanceServerKt", - serverPort.toString() + serverPort.toString(), ) val process = processBuilder.start() @@ -134,14 +129,14 @@ class ConformanceTest { if (!serverReady) { val errorInfo = if (serverErrorOutput.isNotEmpty()) { - "\n\nServer error output:\n${serverErrorOutput}" + "\n\nServer error output:\n$serverErrorOutput" } else { "" } serverProcess?.destroyForcibly() throw IllegalStateException( "Server failed to start within $DEFAULT_SERVER_STARTUP_TIMEOUT_SECONDS seconds. " + - "Check if port $serverPort is available.$errorInfo" + "Check if port $serverPort is available.$errorInfo", ) } @@ -184,11 +179,9 @@ class ConformanceTest { } @TestFactory - fun `MCP Client Conformance Tests`(): List { - return CLIENT_SCENARIOS.map { scenario -> - DynamicTest.dynamicTest("Client: $scenario") { - runClientConformanceTest(scenario) - } + fun `MCP Client Conformance Tests`(): List = CLIENT_SCENARIOS.map { scenario -> + DynamicTest.dynamicTest("Client: $scenario") { + runClientConformanceTest(scenario) } } @@ -197,8 +190,10 @@ class ConformanceTest { "npx", "@modelcontextprotocol/conformance", "server", - "--url", serverUrl, - "--scenario", scenario + "--url", + serverUrl, + "--scenario", + scenario, ).apply { inheritIO() } @@ -211,16 +206,19 @@ class ConformanceTest { val clientCommand = listOf( "java", - "-cp", testClasspath, - "io.modelcontextprotocol.kotlin.sdk.conformance.ConformanceClientKt" + "-cp", + testClasspath, + "io.modelcontextprotocol.kotlin.sdk.conformance.ConformanceClientKt", ) val processBuilder = ProcessBuilder( "npx", "@modelcontextprotocol/conformance", "client", - "--command", clientCommand.joinToString(" "), - "--scenario", scenario + "--command", + clientCommand.joinToString(" "), + "--scenario", + scenario, ).apply { inheritIO() } @@ -228,11 +226,7 @@ class ConformanceTest { runConformanceTest("client", scenario, processBuilder) } - private fun runConformanceTest( - type: String, - scenario: String, - processBuilder: ProcessBuilder - ) { + private fun runConformanceTest(type: String, scenario: String, processBuilder: ProcessBuilder) { logger.info { "Running $type conformance test: $scenario" } val timeoutSeconds = System.getenv("CONFORMANCE_TEST_TIMEOUT_SECONDS")?.toLongOrNull() @@ -242,16 +236,33 @@ class ConformanceTest { val completed = process.waitFor(timeoutSeconds, TimeUnit.SECONDS) if (!completed) { - logger.error { "${type.replaceFirstChar { it.uppercase() }} conformance test '$scenario' timed out after $timeoutSeconds seconds" } + logger.error { + "${type.replaceFirstChar { + it.uppercase() + }} conformance test '$scenario' timed out after $timeoutSeconds seconds" + } process.destroyForcibly() - throw AssertionError("❌ ${type.replaceFirstChar { it.uppercase() }} conformance test '$scenario' timed out after $timeoutSeconds seconds") + throw AssertionError( + "❌ ${type.replaceFirstChar { + it.uppercase() + }} conformance test '$scenario' timed out after $timeoutSeconds seconds", + ) } when (val exitCode = process.exitValue()) { 0 -> logger.info { "✅ ${type.replaceFirstChar { it.uppercase() }} conformance test '$scenario' passed!" } + else -> { - logger.error { "${type.replaceFirstChar { it.uppercase() }} conformance test '$scenario' failed with exit code: $exitCode" } - throw AssertionError("❌ ${type.replaceFirstChar { it.uppercase() }} conformance test '$scenario' failed (exit code: $exitCode). Check test output above for details.") + logger.error { + "${type.replaceFirstChar { + it.uppercase() + }} conformance test '$scenario' failed with exit code: $exitCode" + } + throw AssertionError( + "❌ ${type.replaceFirstChar { + it.uppercase() + }} conformance test '$scenario' failed (exit code: $exitCode). Check test output above for details.", + ) } } } From e9f7f9f0034a9e4ee5f6f8353ee8e2b77ee37d77 Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Fri, 28 Nov 2025 22:31:59 +0200 Subject: [PATCH 06/15] cr --- .../sdk/conformance/ConformanceServer.kt | 8 +++++--- .../kotlin/sdk/conformance/ConformanceTest.kt | 19 ++++++------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt index 419be6c1..8c6c01c5 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt @@ -143,6 +143,8 @@ fun main(args: Array) { sessionReady.complete(Unit) } catch (e: Exception) { logger.error(e) { "Failed to create session" } + serverTransports.remove(newSessionId) + newTransport.close() sessionReady.completeExceptionally(e) } } @@ -208,7 +210,7 @@ private fun createConformanceServer(): Server { ) server.addTool( - name = "test_tool", + name = "test-tool", description = "A test tool for conformance testing", inputSchema = ToolSchema( properties = buildJsonObject { @@ -230,7 +232,7 @@ private fun createConformanceServer(): Server { } server.addResource( - uri = "test://resource", + uri = "test://test-resource", name = "Test Resource", description = "A test resource for conformance testing", mimeType = "text/plain", @@ -243,7 +245,7 @@ private fun createConformanceServer(): Server { } server.addPrompt( - name = "test_prompt", + name = "test-prompt", description = "A test prompt for conformance testing", arguments = listOf( PromptArgument( diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt index a4901d31..65cfc0ff 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt @@ -227,6 +227,7 @@ class ConformanceTest { } private fun runConformanceTest(type: String, scenario: String, processBuilder: ProcessBuilder) { + val capitalizedType = type.replaceFirstChar { it.uppercase() } logger.info { "Running $type conformance test: $scenario" } val timeoutSeconds = System.getenv("CONFORMANCE_TEST_TIMEOUT_SECONDS")?.toLongOrNull() @@ -237,31 +238,23 @@ class ConformanceTest { if (!completed) { logger.error { - "${type.replaceFirstChar { - it.uppercase() - }} conformance test '$scenario' timed out after $timeoutSeconds seconds" + "$capitalizedType conformance test '$scenario' timed out after $timeoutSeconds seconds" } process.destroyForcibly() throw AssertionError( - "❌ ${type.replaceFirstChar { - it.uppercase() - }} conformance test '$scenario' timed out after $timeoutSeconds seconds", + "❌ $capitalizedType conformance test '$scenario' timed out after $timeoutSeconds seconds", ) } when (val exitCode = process.exitValue()) { - 0 -> logger.info { "✅ ${type.replaceFirstChar { it.uppercase() }} conformance test '$scenario' passed!" } + 0 -> logger.info { "✅ $capitalizedType conformance test '$scenario' passed!" } else -> { logger.error { - "${type.replaceFirstChar { - it.uppercase() - }} conformance test '$scenario' failed with exit code: $exitCode" + "$capitalizedType conformance test '$scenario' failed with exit code: $exitCode" } throw AssertionError( - "❌ ${type.replaceFirstChar { - it.uppercase() - }} conformance test '$scenario' failed (exit code: $exitCode). Check test output above for details.", + "❌ $capitalizedType conformance test '$scenario' failed (exit code: $exitCode). Check test output above for details.", ) } } From c4340f9ca3999f436d41862100d7df923386feb8 Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Fri, 28 Nov 2025 22:35:04 +0200 Subject: [PATCH 07/15] cr --- .github/workflows/build.yml | 2 +- .../sdk/conformance/ConformanceClient.kt | 19 ++----------------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a12c911f..30253f99 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,7 +63,7 @@ jobs: run: ./gradlew --no-daemon clean build -Pmcp.kotlin.overrideVersion=0.0.1-SNAPSHOT - name: Run Conformance Tests - run: ./gradlew conformance + run: ./gradlew --no-daemon conformance - name: Upload Conformance Results if: always() diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt index ef9b5267..09ea50bf 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt @@ -18,7 +18,7 @@ private val logger = KotlinLogging.logger {} fun main(args: Array) { require(args.isNotEmpty()) { - "Usage: ConformanceClient " + "Server URL must be provided as an argument" } val serverUrl = args.last() @@ -47,22 +47,7 @@ fun main(args: Array) { val tools = client.listTools() logger.info { "Available tools: ${tools.tools.map { it.name }}" } - val addNumbersTool = tools.tools.find { it.name == "add_numbers" } - if (addNumbersTool != null) { - logger.info { "Calling tool: add_numbers" } - val result = client.callTool( - CallToolRequest( - params = CallToolRequestParams( - name = "add_numbers", - arguments = buildJsonObject { - put("a", JsonPrimitive(5)) - put("b", JsonPrimitive(3)) - }, - ), - ), - ) - logger.info { "Tool result: ${result.content}" } - } else if (tools.tools.isNotEmpty()) { + if (tools.tools.isNotEmpty()) { val toolName = tools.tools.first().name logger.info { "Calling tool: $toolName" } From bfd7b9a5b4f40db4602c238b788978375378262b Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Fri, 28 Nov 2025 22:36:15 +0200 Subject: [PATCH 08/15] cr --- .../kotlin/sdk/conformance/ConformanceServer.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt index 8c6c01c5..12af7df1 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt @@ -315,6 +315,7 @@ private class HttpServerTransport(private val sessionId: String) : AbstractTrans if (response != null) { call.respondText(McpJson.encodeToString(response), ContentType.Application.Json) } else { + pendingResponses.remove(idKey) logger.warn { "Timeout for request $idKey" } call.respondText( McpJson.encodeToString( From 03243a5201794974b9f290d3699d9e29b86e0eef Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Fri, 28 Nov 2025 22:37:57 +0200 Subject: [PATCH 09/15] cr --- .../sdk/conformance/ConformanceServer.kt | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt index 12af7df1..9934bbac 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt @@ -149,9 +149,32 @@ fun main(args: Array) { } } - withTimeoutOrNull(SESSION_CREATION_TIMEOUT_MS) { + val sessionCreated = withTimeoutOrNull(SESSION_CREATION_TIMEOUT_MS) { sessionReady.await() - } ?: logger.warn { "Session creation timed out, proceeding anyway" } + } + + if (sessionCreated == null) { + logger.error { "Session creation timed out" } + serverTransports.remove(newSessionId) + call.respond( + HttpStatusCode.InternalServerError, + jsonFormat.encodeToString( + JsonObject.serializer(), + buildJsonObject { + put("jsonrpc", "2.0") + put( + "error", + buildJsonObject { + put("code", -32000) + put("message", "Session creation timed out") + }, + ) + put("id", JsonNull) + }, + ), + ) + return@post + } newTransport.handleRequest(call, jsonElement) } else { From 8c9e9e1389a7775daa349171b507212ee32a3f99 Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Fri, 28 Nov 2025 22:39:08 +0200 Subject: [PATCH 10/15] cr --- .../kotlin/sdk/conformance/ConformanceTest.kt | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt index 65cfc0ff..17754161 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt +++ b/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt @@ -22,7 +22,8 @@ class ConformanceTest { private var serverProcess: Process? = null private var serverPort: Int by Delegates.notNull() - private val serverErrorOutput = StringBuffer() + private val serverErrorOutput = mutableListOf() + private val maxErrorLines = 500 companion object { private val SERVER_SCENARIOS = listOf( @@ -112,7 +113,12 @@ class ConformanceTest { try { BufferedReader(InputStreamReader(process.errorStream)).use { reader -> reader.lineSequence().forEach { line -> - serverErrorOutput.appendLine(line) + synchronized(serverErrorOutput) { + if (serverErrorOutput.size >= maxErrorLines) { + serverErrorOutput.removeAt(0) + } + serverErrorOutput.add(line) + } logger.debug { "Server stderr: $line" } } } @@ -128,10 +134,12 @@ class ConformanceTest { val serverReady = waitForServerReady(serverUrl) if (!serverReady) { - val errorInfo = if (serverErrorOutput.isNotEmpty()) { - "\n\nServer error output:\n$serverErrorOutput" - } else { - "" + val errorInfo = synchronized(serverErrorOutput) { + if (serverErrorOutput.isNotEmpty()) { + "\n\nServer error output:\n${serverErrorOutput.joinToString("\n")}" + } else { + "" + } } serverProcess?.destroyForcibly() throw IllegalStateException( From 76d8cedfce6430e54f815ebbdc1d3e2adb9565bd Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Mon, 1 Dec 2025 13:07:51 +0200 Subject: [PATCH 11/15] Update .github/workflows/build.yml Co-authored-by: Konstantin Pavlov <1517853+kpavlov@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 30253f99..547811f8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,7 @@ jobs: run: ./gradlew --no-daemon conformance - name: Upload Conformance Results - if: always() + if: ${{ !cancelled() }} uses: actions/upload-artifact@v5 with: name: conformance-results From 498ce26ce7b792e3809c36456867eebf55c4093d Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Mon, 1 Dec 2025 14:52:38 +0200 Subject: [PATCH 12/15] Add support for multiple transports in conformance tests and move to separate module --- .gitignore | 2 +- conformance-test/build.gradle.kts | 38 +++++ .../sdk/conformance/ConformanceClient.kt | 0 .../sdk/conformance/ConformanceServer.kt | 19 +++ .../kotlin/sdk/conformance/ConformanceTest.kt | 130 +++++++++++++----- .../conformance/WebSocketConformanceClient.kt | 104 ++++++++++++++ kotlin-sdk-test/build.gradle.kts | 37 ----- settings.gradle.kts | 1 + 8 files changed, 257 insertions(+), 74 deletions(-) create mode 100644 conformance-test/build.gradle.kts rename {kotlin-sdk-test/src/jvmTest => conformance-test/src/test}/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt (100%) rename {kotlin-sdk-test/src/jvmTest => conformance-test/src/test}/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt (96%) rename {kotlin-sdk-test/src/jvmTest => conformance-test/src/test}/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt (67%) create mode 100644 conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/WebSocketConformanceClient.kt diff --git a/.gitignore b/.gitignore index 384748b7..a8c18ddb 100644 --- a/.gitignore +++ b/.gitignore @@ -58,4 +58,4 @@ dist .junie/ ### Conformance test results ### -kotlin-sdk-test/results/ +conformance-test/results/ diff --git a/conformance-test/build.gradle.kts b/conformance-test/build.gradle.kts new file mode 100644 index 00000000..66e57d15 --- /dev/null +++ b/conformance-test/build.gradle.kts @@ -0,0 +1,38 @@ +import org.gradle.api.tasks.testing.logging.TestExceptionFormat + +plugins { + kotlin("jvm") +} + +dependencies { + testImplementation(project(":kotlin-sdk")) + testImplementation(kotlin("test")) + testImplementation(libs.kotlin.logging) + testImplementation(libs.ktor.client.cio) + testImplementation(libs.ktor.server.cio) + testImplementation(libs.ktor.server.websockets) + testRuntimeOnly(libs.slf4j.simple) +} + +tasks.test { + useJUnitPlatform() + + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showExceptions = true + showCauses = true + showStackTraces = true + exceptionFormat = TestExceptionFormat.FULL + } + + doFirst { + systemProperty("test.classpath", classpath.asPath) + + println("\n" + "=".repeat(60)) + println("MCP CONFORMANCE TESTS") + println("=".repeat(60)) + println("These tests validate compliance with the MCP specification.") + println("=".repeat(60) + "\n") + } +} diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt b/conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt similarity index 100% rename from kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt rename to conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt b/conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt similarity index 96% rename from kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt rename to conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt index 9934bbac..01bdd403 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt +++ b/conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceServer.kt @@ -4,6 +4,7 @@ import io.github.oshai.kotlinlogging.KotlinLogging import io.ktor.http.ContentType import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall +import io.ktor.server.application.install import io.ktor.server.cio.CIO import io.ktor.server.engine.embeddedServer import io.ktor.server.request.header @@ -16,8 +17,11 @@ import io.ktor.server.routing.delete import io.ktor.server.routing.get import io.ktor.server.routing.post import io.ktor.server.routing.routing +import io.ktor.server.websocket.WebSockets +import io.ktor.server.websocket.webSocket import io.modelcontextprotocol.kotlin.sdk.server.Server import io.modelcontextprotocol.kotlin.sdk.server.ServerOptions +import io.modelcontextprotocol.kotlin.sdk.server.WebSocketMcpServerTransport import io.modelcontextprotocol.kotlin.sdk.shared.AbstractTransport import io.modelcontextprotocol.kotlin.sdk.shared.TransportSendOptions import io.modelcontextprotocol.kotlin.sdk.types.CallToolResult @@ -75,7 +79,22 @@ fun main(args: Array) { logger.info { "Starting MCP Conformance Server on port $port" } embeddedServer(CIO, port = port, host = "127.0.0.1") { + install(WebSockets) + routing { + webSocket("/ws") { + logger.info { "WebSocket connection established" } + val transport = WebSocketMcpServerTransport(this) + val server = createConformanceServer() + + try { + server.createSession(transport) + } catch (e: Exception) { + logger.error(e) { "Error in WebSocket session" } + throw e + } + } + get("/mcp") { val sessionId = call.request.header("mcp-session-id") ?: run { diff --git a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt b/conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt similarity index 67% rename from kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt rename to conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt index 17754161..4137943e 100644 --- a/kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt +++ b/conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceTest.kt @@ -17,6 +17,11 @@ import kotlin.properties.Delegates private val logger = KotlinLogging.logger {} +enum class TransportType { + SSE, + WEBSOCKET, +} + @TestInstance(TestInstance.Lifecycle.PER_CLASS) class ConformanceTest { @@ -43,6 +48,17 @@ class ConformanceTest { // "tools-call", ) + private val SERVER_TRANSPORT_TYPES = listOf( + TransportType.SSE, + // TODO: Fix +// TransportType.WEBSOCKET, + ) + + private val CLIENT_TRANSPORT_TYPES = listOf( + TransportType.SSE, + TransportType.WEBSOCKET, + ) + private const val DEFAULT_TEST_TIMEOUT_SECONDS = 30L private const val DEFAULT_SERVER_STARTUP_TIMEOUT_SECONDS = 10 private const val INITIAL_BACKOFF_MS = 50L @@ -176,48 +192,85 @@ class ConformanceTest { } @TestFactory - fun `MCP Server Conformance Tests`(): List { - val serverUrl = "http://127.0.0.1:$serverPort/mcp" - - return SERVER_SCENARIOS.map { scenario -> - DynamicTest.dynamicTest("Server: $scenario") { - runServerConformanceTest(scenario, serverUrl) + fun `MCP Server Conformance Tests`(): List = SERVER_TRANSPORT_TYPES.flatMap { transportType -> + SERVER_SCENARIOS.map { scenario -> + DynamicTest.dynamicTest("Server [$transportType]: $scenario") { + runServerConformanceTest(scenario, transportType) } } } @TestFactory - fun `MCP Client Conformance Tests`(): List = CLIENT_SCENARIOS.map { scenario -> - DynamicTest.dynamicTest("Client: $scenario") { - runClientConformanceTest(scenario) + fun `MCP Client Conformance Tests`(): List = CLIENT_TRANSPORT_TYPES.flatMap { transportType -> + CLIENT_SCENARIOS.map { scenario -> + DynamicTest.dynamicTest("Client [$transportType]: $scenario") { + runClientConformanceTest(scenario, transportType) + } } } - private fun runServerConformanceTest(scenario: String, serverUrl: String) { - val processBuilder = ProcessBuilder( - "npx", - "@modelcontextprotocol/conformance", - "server", - "--url", - serverUrl, - "--scenario", - scenario, - ).apply { - inheritIO() + private fun runServerConformanceTest(scenario: String, transportType: TransportType) { + val processBuilder = when (transportType) { + TransportType.SSE -> { + val serverUrl = "http://127.0.0.1:$serverPort/mcp" + ProcessBuilder( + "npx", + "@modelcontextprotocol/conformance", + "server", + "--url", + serverUrl, + "--scenario", + scenario, + ).apply { + inheritIO() + } + } + + TransportType.WEBSOCKET -> { + val serverUrl = "ws://127.0.0.1:$serverPort/ws" + ProcessBuilder( + "npx", + "@modelcontextprotocol/conformance", + "server", + "--url", + serverUrl, + "--scenario", + scenario, + ).apply { + inheritIO() + } + } } - runConformanceTest("server", scenario, processBuilder) + runConformanceTest("server", scenario, processBuilder, transportType) } - private fun runClientConformanceTest(scenario: String) { + private fun runClientConformanceTest(scenario: String, transportType: TransportType) { val testClasspath = getTestClasspath() - val clientCommand = listOf( - "java", - "-cp", - testClasspath, - "io.modelcontextprotocol.kotlin.sdk.conformance.ConformanceClientKt", - ) + val clientCommand = when (transportType) { + TransportType.SSE -> { + val serverUrl = "http://127.0.0.1:$serverPort/mcp" + listOf( + "java", + "-cp", + testClasspath, + "io.modelcontextprotocol.kotlin.sdk.conformance.ConformanceClientKt", + serverUrl, + ) + } + + TransportType.WEBSOCKET -> { + val serverUrl = "ws://127.0.0.1:$serverPort/ws" + listOf( + "java", + "-cp", + testClasspath, + "io.modelcontextprotocol.kotlin.sdk.conformance.WebSocketConformanceClientKt", + serverUrl, + ) + } + } val processBuilder = ProcessBuilder( "npx", @@ -231,12 +284,17 @@ class ConformanceTest { inheritIO() } - runConformanceTest("client", scenario, processBuilder) + runConformanceTest("client", scenario, processBuilder, transportType) } - private fun runConformanceTest(type: String, scenario: String, processBuilder: ProcessBuilder) { + private fun runConformanceTest( + type: String, + scenario: String, + processBuilder: ProcessBuilder, + transportType: TransportType, + ) { val capitalizedType = type.replaceFirstChar { it.uppercase() } - logger.info { "Running $type conformance test: $scenario" } + logger.info { "Running $type conformance test [$transportType]: $scenario" } val timeoutSeconds = System.getenv("CONFORMANCE_TEST_TIMEOUT_SECONDS")?.toLongOrNull() ?: DEFAULT_TEST_TIMEOUT_SECONDS @@ -246,23 +304,23 @@ class ConformanceTest { if (!completed) { logger.error { - "$capitalizedType conformance test '$scenario' timed out after $timeoutSeconds seconds" + "$capitalizedType conformance test [$transportType] '$scenario' timed out after $timeoutSeconds seconds" } process.destroyForcibly() throw AssertionError( - "❌ $capitalizedType conformance test '$scenario' timed out after $timeoutSeconds seconds", + "❌ $capitalizedType conformance test [$transportType] '$scenario' timed out after $timeoutSeconds seconds", ) } when (val exitCode = process.exitValue()) { - 0 -> logger.info { "✅ $capitalizedType conformance test '$scenario' passed!" } + 0 -> logger.info { "✅ $capitalizedType conformance test [$transportType] '$scenario' passed!" } else -> { logger.error { - "$capitalizedType conformance test '$scenario' failed with exit code: $exitCode" + "$capitalizedType conformance test [$transportType] '$scenario' failed with exit code: $exitCode" } throw AssertionError( - "❌ $capitalizedType conformance test '$scenario' failed (exit code: $exitCode). Check test output above for details.", + "❌ $capitalizedType conformance test [$transportType] '$scenario' failed (exit code: $exitCode). Check test output above for details.", ) } } diff --git a/conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/WebSocketConformanceClient.kt b/conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/WebSocketConformanceClient.kt new file mode 100644 index 00000000..f1123192 --- /dev/null +++ b/conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/WebSocketConformanceClient.kt @@ -0,0 +1,104 @@ +package io.modelcontextprotocol.kotlin.sdk.conformance + +import io.github.oshai.kotlinlogging.KotlinLogging +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.websocket.WebSockets +import io.ktor.client.plugins.websocket.webSocket +import io.ktor.websocket.WebSocketSession +import io.modelcontextprotocol.kotlin.sdk.client.Client +import io.modelcontextprotocol.kotlin.sdk.shared.MCP_SUBPROTOCOL +import io.modelcontextprotocol.kotlin.sdk.shared.WebSocketMcpTransport +import io.modelcontextprotocol.kotlin.sdk.types.CallToolRequest +import io.modelcontextprotocol.kotlin.sdk.types.CallToolRequestParams +import io.modelcontextprotocol.kotlin.sdk.types.Implementation +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject + +private val logger = KotlinLogging.logger {} + +class WebSocketClientTransport(override val session: WebSocketSession) : WebSocketMcpTransport() { + override suspend fun initializeSession() { + logger.debug { "WebSocket client session initialized" } + } +} + +fun main(args: Array) { + require(args.isNotEmpty()) { + "Server WebSocket URL must be provided as an argument" + } + + val serverUrl = args.last() + logger.info { "Connecting to WebSocket test server at: $serverUrl" } + + val httpClient = HttpClient(CIO) { + install(WebSockets) + } + + var exitCode = 0 + + runBlocking { + try { + httpClient.webSocket(serverUrl, request = { + headers.append("Sec-WebSocket-Protocol", MCP_SUBPROTOCOL) + }) { + val transport = WebSocketClientTransport(this) + + val client = Client( + clientInfo = Implementation( + name = "kotlin-conformance-client-websocket", + version = "1.0.0", + ), + ) + + try { + client.connect(transport) + logger.info { "✅ Connected to server successfully" } + + try { + val tools = client.listTools() + logger.info { "Available tools: ${tools.tools.map { it.name }}" } + + if (tools.tools.isNotEmpty()) { + val toolName = tools.tools.first().name + logger.info { "Calling tool: $toolName" } + + val result = client.callTool( + CallToolRequest( + params = CallToolRequestParams( + name = toolName, + arguments = buildJsonObject { + put("input", JsonPrimitive("test")) + }, + ), + ), + ) + logger.info { "Tool result: ${result.content}" } + } + } catch (e: Exception) { + logger.debug(e) { "Error during tool operations (may be expected for some scenarios)" } + } + + logger.info { "✅ Client operations completed successfully" } + } catch (e: Exception) { + logger.error(e) { "❌ Client failed" } + exitCode = 1 + } finally { + try { + transport.close() + } catch (e: Exception) { + logger.warn(e) { "Error closing transport" } + } + } + } + } catch (e: Exception) { + logger.error(e) { "❌ WebSocket connection failed" } + exitCode = 1 + } finally { + httpClient.close() + } + } + + kotlin.system.exitProcess(exitCode) +} diff --git a/kotlin-sdk-test/build.gradle.kts b/kotlin-sdk-test/build.gradle.kts index caf88b23..252d0e21 100644 --- a/kotlin-sdk-test/build.gradle.kts +++ b/kotlin-sdk-test/build.gradle.kts @@ -1,6 +1,3 @@ -import org.gradle.api.tasks.testing.logging.TestExceptionFormat -import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJvmCompilation - plugins { id("mcp.multiplatform") } @@ -34,37 +31,3 @@ kotlin { } } } - -tasks.register("conformance") { - group = "conformance" - description = "Run MCP conformance tests with detailed output" - - val jvmCompilation = kotlin.targets["jvm"].compilations["test"] as KotlinJvmCompilation - testClassesDirs = jvmCompilation.output.classesDirs - classpath = jvmCompilation.runtimeDependencyFiles - - useJUnitPlatform() - - filter { - includeTestsMatching("*ConformanceTest*") - } - - testLogging { - events("passed", "skipped", "failed") - showStandardStreams = true - showExceptions = true - showCauses = true - showStackTraces = true - exceptionFormat = TestExceptionFormat.FULL - } - - doFirst { - systemProperty("test.classpath", classpath.asPath) - - println("\n" + "=".repeat(60)) - println("MCP CONFORMANCE TESTS") - println("=".repeat(60)) - println("These tests validate compliance with the MCP specification.") - println("=".repeat(60) + "\n") - } -} diff --git a/settings.gradle.kts b/settings.gradle.kts index 368bff38..d7ec54f8 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -23,4 +23,5 @@ include( ":kotlin-sdk-server", ":kotlin-sdk", ":kotlin-sdk-test", + ":conformance-test", ) From df6f844e331557b3669e8b4187ae6f040e7bb189 Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Mon, 1 Dec 2025 16:55:42 +0200 Subject: [PATCH 13/15] cr --- .github/workflows/build.yml | 34 --------------------- .github/workflows/conformance.yml | 50 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/conformance.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 547811f8..f3e4d167 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,11 +28,6 @@ jobs: java-version: '21' distribution: 'temurin' - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - name: Setup Gradle uses: gradle/actions/setup-gradle@v5 with: @@ -62,35 +57,6 @@ jobs: working-directory: ./samples/weather-stdio-server run: ./gradlew --no-daemon clean build -Pmcp.kotlin.overrideVersion=0.0.1-SNAPSHOT - - name: Run Conformance Tests - run: ./gradlew --no-daemon conformance - - - name: Upload Conformance Results - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v5 - with: - name: conformance-results - path: kotlin-sdk-test/results/ - - - name: Upload Reports - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v5 - with: - name: reports - path: | - **/build/reports/ - - - name: Publish Test Report - uses: mikepenz/action-junit-report@v6 - if: ${{ !cancelled() }} # always run even if the previous step fails - with: - report_paths: '**/test-results/**/TEST-*.xml' - detailed_summary: true - flaky_summary: true - include_empty_in_summary: false - include_time_in_summary: true - annotate_only: true - - name: Disable Auto-Merge on Fail if: failure() && github.event_name == 'pull_request' run: gh pr merge --disable-auto "$PR_URL" diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 00000000..c736ad3d --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,50 @@ +name: Conformance Tests + +on: + workflow_dispatch: + pull_request: + branches: [ main ] + push: + branches: [ main ] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # Cancel only when the run is NOT on `main` branch + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + run-conformance: + runs-on: macos-latest-xlarge + name: Run Conformance Tests + timeout-minutes: 20 + env: + JAVA_OPTS: "-Xmx8g -Dfile.encoding=UTF-8 -Djava.awt.headless=true -Dkotlin.daemon.jvm.options=-Xmx6g" + steps: + - uses: actions/checkout@v6 + + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + java-version: '21' + distribution: 'temurin' + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '24.11.1' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v5 + with: + add-job-summary: 'always' + cache-read-only: true + + - name: Run Conformance Tests + run: ./gradlew --no-daemon :conformance-test:test + + - name: Upload Conformance Results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v5 + with: + name: conformance-results + path: conformance-test/results/ From b911d1cb55a401cccc73146076a703b2756ecd22 Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Mon, 1 Dec 2025 17:02:23 +0200 Subject: [PATCH 14/15] cr --- .github/workflows/build.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f3e4d167..9eed688a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,6 +57,25 @@ jobs: working-directory: ./samples/weather-stdio-server run: ./gradlew --no-daemon clean build -Pmcp.kotlin.overrideVersion=0.0.1-SNAPSHOT + - name: Upload Reports + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v5 + with: + name: reports + path: | + **/build/reports/ + + - name: Publish Test Report + uses: mikepenz/action-junit-report@v6 + if: ${{ !cancelled() }} # always run even if the previous step fails + with: + report_paths: '**/test-results/**/TEST-*.xml' + detailed_summary: true + flaky_summary: true + include_empty_in_summary: false + include_time_in_summary: true + annotate_only: true + - name: Disable Auto-Merge on Fail if: failure() && github.event_name == 'pull_request' run: gh pr merge --disable-auto "$PR_URL" From 7ab4b9835e44c94c029053292af93c64428dc4de Mon Sep 17 00:00:00 2001 From: Sergey Karpov Date: Mon, 1 Dec 2025 17:40:54 +0200 Subject: [PATCH 15/15] cr --- .github/workflows/build.yml | 1 + .github/workflows/gradle-publish.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9eed688a..719f9c7a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,6 +41,7 @@ jobs: clean \ ktlintCheck \ build \ + -x :conformance-test:test \ koverLog koverHtmlReport \ publishToMavenLocal \ -Pversion=0.0.1-SNAPSHOT diff --git a/.github/workflows/gradle-publish.yml b/.github/workflows/gradle-publish.yml index d9b28012..bb81dbe5 100644 --- a/.github/workflows/gradle-publish.yml +++ b/.github/workflows/gradle-publish.yml @@ -34,7 +34,7 @@ jobs: uses: gradle/actions/setup-gradle@v5 - name: Clean Build with Gradle - run: ./gradlew clean build + run: ./gradlew clean build -x :conformance-test:test - name: Publish to Maven Central Portal id: publish