Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pre compute suggestion db during build time #5698

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
b95a302
build: indexStdLib
4e6 Feb 9, 2023
6bb4d43
feat: serialization manager
4e6 Feb 14, 2023
9c84027
feat: update EnsureCompiledJob
4e6 Feb 15, 2023
b84603e
feat: simplify suggestions init
4e6 Feb 15, 2023
1ee00d3
misc: cleanup
4e6 Feb 20, 2023
e2ac361
Serialize provided suggestions to each standard library
JaroslavTulach Feb 20, 2023
d62ec42
Demo of iteratively adding constructing Tree
JaroslavTulach Feb 21, 2023
560a107
Merge remote-tracking branch 'origin/develop' into wip/db/5068-pre-co…
JaroslavTulach Feb 22, 2023
b68da75
Only regenerate index when necessary
hubertp Feb 22, 2023
8f7314e
Merge branch 'wip/db/5068-pre-compute-suggestion-db-during-build-time…
JaroslavTulach Feb 22, 2023
2fbdf75
Using Jackson to serialize and deserialize Suggestion.Module
JaroslavTulach Feb 23, 2023
f413b62
Can serialize Suggestion[]
JaroslavTulach Feb 23, 2023
958cc78
Can serialize record with List<Suggestion>
JaroslavTulach Feb 23, 2023
1ff4c25
Serialize and deserialize the Suggestion list as a case class
JaroslavTulach Feb 24, 2023
ce821b1
Can load back the Tree.Node[Suggestion] from a cache
JaroslavTulach Feb 24, 2023
5bff1ab
Less debug messages
JaroslavTulach Feb 24, 2023
f480e03
subscribe to Api.LibraryLoaded
JaroslavTulach Feb 24, 2023
335305a
Merge branch 'develop' into wip/db/5068-pre-compute-suggestion-db-dur…
4e6 Mar 2, 2023
b4a98a4
feat: suggestions cache
4e6 Mar 2, 2023
6638c9d
feat: suggestions deserialization
4e6 Mar 5, 2023
283b6f5
feat: generate docs
4e6 Mar 6, 2023
9081707
misc: format
4e6 Mar 6, 2023
0e769c1
fix: suggestions loading
4e6 Mar 6, 2023
6a96e38
misc: run deserialization in background
4e6 Mar 6, 2023
77c5a2b
test: fix language-server
4e6 Mar 6, 2023
009b595
test: fix runtime
4e6 Mar 6, 2023
31773a6
Merge branch 'develop' into wip/db/5068-pre-compute-suggestion-db-dur…
4e6 Mar 6, 2023
a45898a
test: fix RuntimeServerTest
4e6 Mar 6, 2023
5314e10
test: fix RuntimeErrorsTest
4e6 Mar 6, 2023
1447ed0
test: fix RuntimeInstrumentTest
4e6 Mar 6, 2023
20c2e40
test: fix RuntimeSuggestionUpdatesTest
4e6 Mar 6, 2023
89739c0
test: fix RuntimeStdlibTest
4e6 Mar 6, 2023
36862c6
test: fix RuntimeComponentsTest
4e6 Mar 6, 2023
3468399
misc: cleanup
4e6 Mar 6, 2023
952f0f6
Merge branch 'develop' into wip/db/5068-pre-compute-suggestion-db-dur…
mergify[bot] Mar 8, 2023
4742f5f
misc: review comments
4e6 Mar 8, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,20 @@ import java.util.logging.Level
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.ClassTagExtensions
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.core.JsonFactory
import java.nio.charset.StandardCharsets

final class SuggestionsSerializationManager(logger: TruffleLogger) {
JaroslavTulach marked this conversation as resolved.
Show resolved Hide resolved

/** The debug logging level. */
private val debugLogLevel = Level.FINE

private lazy val mapper = {
val factory = new JsonFactory()
val mapper = new ObjectMapper(factory) with ClassTagExtensions
val mapper = new ObjectMapper() with ClassTagExtensions
mapper.registerModule(DefaultScalaModule)
}

case class SuggetionWithVersion (version : Int, suggestions : Iterable[Suggestion])

def serialize(
suggestions: Iterable[Suggestion],
pkg: Package[TruffleFile]
Expand All @@ -37,19 +38,29 @@ final class SuggestionsSerializationManager(logger: TruffleLogger) {

r.createDirectories()
f = r.resolve("suggestion.json")
val json = mapper.writeValueAsString(suggestions)
val w = f.newBufferedWriter()
w.write(json)
val w = f.newBufferedWriter(StandardCharsets.UTF_8)
w.write(toJson(suggestions))
w.close()
} finally {
System.err.println("Out in " + f)
}

}

def toJson(suggestions: Iterable[Suggestion]): String = new String(mapper
.writerWithDefaultPrettyPrinter()
.writeValueAsBytes(new SuggetionWithVersion(5068, suggestions)))

def deserialize(pkg: Package[TruffleFile]): Unit = {
logger.log(debugLogLevel, "deserialize [{}]", pkg.config.name)
throw new UnsupportedOperationException("deserialize: " + pkg.root)
var f: TruffleFile = null
try {
val r: TruffleFile = pkg.root.resolve(pkg.config.name)
r.createDirectories()
f = r.resolve("suggestion.json")
new String(f.readAllBytes, StandardCharsets.UTF_8)
} finally {
System.err.println("Loaded from " + f)
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ package org.enso.compiler.context
import org.enso.polyglot.Suggestion
import org.enso.polyglot.data.{These, Tree}
import org.enso.polyglot.runtime.Runtime.Api
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.ClassTagExtensions
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.databind.ObjectReader

object SuggestionDiff {

Expand All @@ -28,6 +32,20 @@ object SuggestionDiff {
true
}

case class SuggestionData(version : Int, suggestions : Iterable[Suggestion])

def deserialize(
json: String
): Tree[Api.SuggestionUpdate] = {
val mapper = new ObjectMapper() with ClassTagExtensions
mapper.registerModule(DefaultScalaModule)

val r : ObjectReader = mapper.readerFor[SuggestionData]
val o : SuggestionData = r.readValue[SuggestionData](json)
System.err.println("suggestions are here: " + o.suggestions)
JaroslavTulach marked this conversation as resolved.
Show resolved Hide resolved
Tree.empty
}

/** Compare two suggestions for equality.
*
* It is used to find a matching suggestion when joining the previous and
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package org.enso.compiler.test.context

import org.enso.compiler.context.SuggestionBuilder
import org.enso.compiler.context.SuggestionDiff
import org.enso.compiler.core.IR
import org.enso.interpreter.runtime
import org.enso.interpreter.runtime.EnsoContext
import org.enso.interpreter.test.InterpreterContext
import org.enso.pkg.QualifiedName
import org.enso.polyglot.{LanguageInfo, MethodNames, Suggestion}
import org.enso.polyglot.data.Tree
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpecLike

class SuggestionDiffTest extends AnyWordSpecLike with Matchers {
private val ctx = new InterpreterContext()
private val langCtx = ctx.ctx
.getBindings(LanguageInfo.ID)
.invokeMember(MethodNames.TopScope.LEAK_CONTEXT)
.asHostObject[EnsoContext]()

implicit private class PreprocessModule(code: String) {

def preprocessModule(name: QualifiedName): IR.Module = {
val module = new runtime.Module(
name,
null,
code.stripMargin.linesIterator.mkString("\n")
)
langCtx.getCompiler.run(module)
module.getIr
}

def preprocessModule: IR.Module =
preprocessModule(Module)

}

private val Module = QualifiedName(List("Unnamed"), "Test")
private val ModuleNode = Tree.Node(
Suggestion.Module(
module = Module.toString,
documentation = None
),
Vector()
)

@annotation.nowarn
def endOfLine(line: Int, character: Int): Suggestion.Position =
Suggestion.Position(line + 1, 0)

"build method without explicit arguments" in {
val ssm = new org.enso.compiler.SuggestionsSerializationManager(null)

val expected = Tree.Root(
Vector(
ModuleNode,
Tree.Node(
Suggestion.Method(
externalId = None,
module = "Unnamed.Test",
name = "foo",
arguments = Seq(
Suggestion.Argument("self", "Unnamed.Test", false, false, None)
),
selfType = "Unnamed.Test",
returnType = SuggestionBuilder.Any,
isStatic = true,
documentation = None
),
Vector()
)
)
)

val code = """foo = 42"""
val module = code.preprocessModule
val t1 = build(code, module)

t1 shouldEqual expected

val data = t1.toVector
val json = ssm.toJson(data)

System.out.println(json)

val t2 = SuggestionDiff.deserialize(json);
t2 shouldEqual expected
}
private def build(
source: String,
ir: IR.Module,
module: QualifiedName = Module
): Tree.Root[Suggestion] =
SuggestionBuilder(source).build(module, ir)
}