Skip to content

Commit

Permalink
Merge #7038
Browse files Browse the repository at this point in the history 
7038: PERF: speedup proc macro tt json serialization/deserialization r=vlad20012 a=vlad20012

1. Use `Jackson` instead of `Gson`. `Jackson` is faster and is recommended in IntelliJ platform and plugins
2. Use hand-written stream serializers/deserializers

This speeds up serialization up to x10 and deserialization up to x30

Co-authored-by: vlad20012 <beskvlad@gmail.com>
  • Loading branch information
@bors @vlad20012
bors[bot] and vlad20012 committed Apr 4, 2021
2 parents 8fd4604 + 9c5d4e2 commit 2e94622
Show file tree
Hide file tree
Showing 12 changed files with 403 additions and 113 deletions.
3 changes: 0 additions & 3 deletions src/main/kotlin/org/rust/lang/core/macros/proc/ProcMacroExpander.kt
View file
Expand Up @@ -5,7 +5,6 @@

package org.rust.lang.core.macros.proc

import com.google.gson.JsonParseException
import com.intellij.openapi.project.Project
import org.rust.lang.core.macros.*
import org.rust.lang.core.macros.tt.MappedSubtree
Expand Down Expand Up @@ -57,8 +56,6 @@ class ProcMacroExpander(
return Err(ProcMacroExpansionError.CantRunExpander)
} catch (e: IOException) {
return Err(ProcMacroExpansionError.ExceptionThrown(e))
} catch (e: JsonParseException) {
return Err(ProcMacroExpansionError.ExceptionThrown(e))
}
return when (response) {
is Response.ExpansionMacro -> Ok(response.expansion)
Expand Down
49 changes: 32 additions & 17 deletions src/main/kotlin/org/rust/lang/core/macros/proc/ProcMacroServerPool.kt
View file
Expand Up @@ -5,9 +5,13 @@

package org.rust.lang.core.macros.proc

import com.fasterxml.jackson.core.JsonGenerator
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.module.SimpleModule
import com.fasterxml.jackson.module.kotlin.KotlinModule
import com.google.common.annotations.VisibleForTesting
import com.google.gson.GsonBuilder
import com.google.gson.JsonParseException
import com.intellij.execution.process.ProcessIOExecutorService
import com.intellij.openapi.Disposable
import com.intellij.openapi.diagnostic.debug
Expand All @@ -17,7 +21,8 @@ import com.intellij.openapi.util.registry.Registry
import com.intellij.util.concurrency.AppExecutorUtil
import org.rust.lang.core.macros.MACRO_LOG
import org.rust.lang.core.macros.tt.TokenTree
import org.rust.lang.core.macros.tt.TokenTreeJsonAdapter
import org.rust.lang.core.macros.tt.TokenTreeJsonDeserializer
import org.rust.lang.core.macros.tt.TokenTreeJsonSerializer
import org.rust.openapiext.RsPathManager
import org.rust.stdext.*
import java.io.*
Expand All @@ -40,11 +45,11 @@ class ProcMacroServerPool private constructor(
Disposer.register(parentDisposable, pool)
}

@Throws(ProcessCreationException::class, IOException::class, JsonParseException::class, TimeoutException::class)
@Throws(ProcessCreationException::class, IOException::class, TimeoutException::class)
fun send(request: Request): Response {
val io = pool.alloc() // Throws ProcessCreationException
return try {
io.send(request) // Throws IOException, JsonParseException, TimeoutException
io.send(request) // Throws IOException, TimeoutException
} finally {
pool.free(io)
}
Expand Down Expand Up @@ -173,7 +178,7 @@ private class ProcMacroServerProcess private constructor(
@Volatile
private var isDisposed: Boolean = false

@Throws(IOException::class, JsonParseException::class, TimeoutException::class)
@Throws(IOException::class, TimeoutException::class)
fun send(request: Request): Response {
if (!lock.tryLock()) error("`send` must not be called from multiple threads simultaneously")
return try {
Expand Down Expand Up @@ -231,16 +236,15 @@ private class ProcMacroServerProcess private constructor(
}
}

@Throws(IOException::class, JsonParseException::class)
@Throws(IOException::class)
private fun writeAndRead(request: Request): Response {
stdin.write(gson.toJson(request, Request::class.java))
ProcMacroJsonParser.jackson.writeValue(stdin, request)
stdin.write("\n")
stdin.flush()

stdout.skipUntilJsonObject()

return gson.fromJson(gson.newJsonReader(stdout), Response::class.java)
?: throw EOFException()
return ProcMacroJsonParser.jackson.readValue(stdout, Response::class.java)
}

/**
Expand Down Expand Up @@ -272,13 +276,6 @@ private class ProcMacroServerProcess private constructor(
}

companion object {
private val gson = GsonBuilder()
.serializeNulls()
.registerTypeAdapter(Request::class.java, RequestJsonAdapter())
.registerTypeAdapter(Response::class.java, ResponseJsonAdapter())
.registerTypeAdapter(TokenTree::class.java, TokenTreeJsonAdapter())
.create()

@Throws(ProcessCreationException::class)
fun createAndRun(expanderExecutable: Path): ProcMacroServerProcess {
MACRO_LOG.debug { "Starting proc macro expander process $expanderExecutable" }
Expand All @@ -302,3 +299,21 @@ private class ProcMacroServerProcess private constructor(
}
}
}

@VisibleForTesting
object ProcMacroJsonParser {
val jackson: ObjectMapper = ObjectMapper()
.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false)
.configure(JsonGenerator.Feature.FLUSH_PASSED_TO_STREAM, false)
.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false)
.configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, false)
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.registerModule(KotlinModule())
.registerModule(
SimpleModule()
.addSerializer(Request::class.java, RequestJsonSerializer())
.addDeserializer(Response::class.java, ResponseJsonDeserializer())
.addDeserializer(TokenTree::class.java, TokenTreeJsonDeserializer)
.addSerializer(TokenTree::class.java, TokenTreeJsonSerializer)
)
}
36 changes: 22 additions & 14 deletions src/main/kotlin/org/rust/lang/core/macros/proc/Request.kt
View file
Expand Up @@ -5,34 +5,42 @@

package org.rust.lang.core.macros.proc

import com.google.gson.JsonElement
import com.google.gson.JsonObject
import com.google.gson.JsonSerializationContext
import com.google.gson.JsonSerializer
import com.google.gson.annotations.SerializedName
import com.fasterxml.jackson.core.JsonGenerator
import com.fasterxml.jackson.databind.SerializerProvider
import org.rust.lang.core.macros.tt.TokenTree
import java.lang.reflect.Type
import org.rust.lang.core.macros.tt.TokenTreeJsonSerializer
import org.rust.stdext.exhaustive
import org.rust.util.RsJacksonSerializer

// This is a sealed class because there is `ListMacro` request kind which we don't use for now
sealed class Request {
// data class ListMacro(...)
data class ExpansionMacro(
@SerializedName("macro_body")
val macroBody: TokenTree.Subtree,
@SerializedName("macro_name")
val macroName: String,
val attributes: TokenTree.Subtree?,
val lib: String,
val env: List<List<String>>
) : Request()
}

class RequestJsonAdapter : JsonSerializer<Request> {
override fun serialize(json: Request, type: Type, context: JsonSerializationContext): JsonElement {
return when (json) {
is Request.ExpansionMacro -> JsonObject().apply {
add("ExpansionMacro", context.serialize(json, json.javaClass))
class RequestJsonSerializer : RsJacksonSerializer<Request>(Request::class.java) {
override fun serialize(request: Request, gen: JsonGenerator, provider: SerializerProvider) {
when (request) {
is Request.ExpansionMacro -> gen.writeJsonObjectWithSingleField("ExpansionMacro") {
writeJsonObject {
writeField("macro_body") { TokenTreeJsonSerializer.writeSubtree(request.macroBody, gen) }
writeStringField("macro_name", request.macroName)
writeNullableField("attributes", request.attributes) { attributes ->
TokenTreeJsonSerializer.writeSubtree(attributes, gen)
}
writeStringField("lib", request.lib)
writeArrayField("env", request.env) { list ->
writeArray(list) { writeString(it) }
}
}
}
}
}.exhaustive
}

}
34 changes: 21 additions & 13 deletions src/main/kotlin/org/rust/lang/core/macros/proc/Response.kt
View file
Expand Up @@ -5,26 +5,34 @@

package org.rust.lang.core.macros.proc

import com.google.gson.JsonDeserializationContext
import com.google.gson.JsonDeserializer
import com.google.gson.JsonElement
import com.google.gson.JsonParseException
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.core.JsonProcessingException
import com.fasterxml.jackson.databind.DeserializationContext
import org.rust.lang.core.macros.tt.TokenTree
import java.lang.reflect.Type
import org.rust.lang.core.macros.tt.TokenTreeJsonDeserializer
import org.rust.util.RsJacksonDeserializer
import java.io.IOException

sealed class Response {
data class Error(val message: String) : Response()
data class ExpansionMacro(val expansion: TokenTree.Subtree) : Response()
}

class ResponseJsonAdapter : JsonDeserializer<Response> {
@Throws(JsonParseException::class)
override fun deserialize(json: JsonElement, type: Type, context: JsonDeserializationContext): Response? {
val obj = json.asJsonObject
return when {
obj.has("Error") -> context.deserialize(obj["Error"], Response.Error::class.java)
obj.has("ExpansionMacro") -> context.deserialize(obj["ExpansionMacro"], Response.ExpansionMacro::class.java)
else -> null
class ResponseJsonDeserializer : RsJacksonDeserializer<Response>(Response::class.java) {
@Throws(IOException::class, JsonProcessingException::class)
override fun deserialize(parser: JsonParser, context: DeserializationContext): Response {
return context.readSingleFieldObject { key ->
when (key) {
"Error" -> context.readValue<Response.Error>()
"ExpansionMacro" -> {
val r = context.readSingleFieldObject {
if (it != "expansion") context.reportInputMismatch("Unknown field `$it`, `expansion` expected")
TokenTreeJsonDeserializer.readSubtree(context)
}
Response.ExpansionMacro(r)
}
else -> context.reportInputMismatch("Unknown response kind `$key`")
}
}
}
}
2 changes: 0 additions & 2 deletions src/main/kotlin/org/rust/lang/core/macros/tt/TokenTree.kt
View file
Expand Up @@ -5,7 +5,6 @@

package org.rust.lang.core.macros.tt

import com.google.gson.annotations.SerializedName
import org.rust.lang.core.macros.tt.Spacing.Alone
import org.rust.lang.core.psi.MacroBraces

Expand Down Expand Up @@ -38,7 +37,6 @@ sealed class TokenTree {

data class Subtree(
val delimiter: Delimiter?,
@SerializedName("token_trees")
val tokenTrees: List<TokenTree>
): TokenTree()
}
Expand Down
51 changes: 0 additions & 51 deletions src/main/kotlin/org/rust/lang/core/macros/tt/TokenTreeJsonAdapter.kt
View file

This file was deleted.

0 comments on commit 2e94622

Please sign in to comment.