Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Pre compute suggestion db during build time #5698

Merged
merged 36 commits into from
Mar 8, 2023
Merged

Pre compute suggestion db during build time #5698

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
b95a302
build: indexStdLib
4e6 Feb 9, 2023
6bb4d43
feat: serialization manager
4e6 Feb 14, 2023
9c84027
feat: update EnsureCompiledJob
4e6 Feb 15, 2023
b84603e
feat: simplify suggestions init
4e6 Feb 15, 2023
1ee00d3
misc: cleanup
4e6 Feb 20, 2023
e2ac361
Serialize provided suggestions to each standard library
JaroslavTulach Feb 20, 2023
d62ec42
Demo of iteratively adding constructing Tree
JaroslavTulach Feb 21, 2023
560a107
Merge remote-tracking branch 'origin/develop' into wip/db/5068-pre-co…
JaroslavTulach Feb 22, 2023
b68da75
Only regenerate index when necessary
hubertp Feb 22, 2023
8f7314e
Merge branch 'wip/db/5068-pre-compute-suggestion-db-during-build-time…
JaroslavTulach Feb 22, 2023
2fbdf75
Using Jackson to serialize and deserialize Suggestion.Module
JaroslavTulach Feb 23, 2023
f413b62
Can serialize Suggestion[]
JaroslavTulach Feb 23, 2023
958cc78
Can serialize record with List<Suggestion>
JaroslavTulach Feb 23, 2023
1ff4c25
Serialize and deserialize the Suggestion list as a case class
JaroslavTulach Feb 24, 2023
ce821b1
Can load back the Tree.Node[Suggestion] from a cache
JaroslavTulach Feb 24, 2023
5bff1ab
Less debug messages
JaroslavTulach Feb 24, 2023
f480e03
subscribe to Api.LibraryLoaded
JaroslavTulach Feb 24, 2023
335305a
Merge branch 'develop' into wip/db/5068-pre-compute-suggestion-db-dur…
4e6 Mar 2, 2023
b4a98a4
feat: suggestions cache
4e6 Mar 2, 2023
6638c9d
feat: suggestions deserialization
4e6 Mar 5, 2023
283b6f5
feat: generate docs
4e6 Mar 6, 2023
9081707
misc: format
4e6 Mar 6, 2023
0e769c1
fix: suggestions loading
4e6 Mar 6, 2023
6a96e38
misc: run deserialization in background
4e6 Mar 6, 2023
77c5a2b
test: fix language-server
4e6 Mar 6, 2023
009b595
test: fix runtime
4e6 Mar 6, 2023
31773a6
Merge branch 'develop' into wip/db/5068-pre-compute-suggestion-db-dur…
4e6 Mar 6, 2023
a45898a
test: fix RuntimeServerTest
4e6 Mar 6, 2023
5314e10
test: fix RuntimeErrorsTest
4e6 Mar 6, 2023
1447ed0
test: fix RuntimeInstrumentTest
4e6 Mar 6, 2023
20c2e40
test: fix RuntimeSuggestionUpdatesTest
4e6 Mar 6, 2023
89739c0
test: fix RuntimeStdlibTest
4e6 Mar 6, 2023
36862c6
test: fix RuntimeComponentsTest
4e6 Mar 6, 2023
3468399
misc: cleanup
4e6 Mar 6, 2023
952f0f6
Merge branch 'develop' into wip/db/5068-pre-compute-suggestion-db-dur…
mergify[bot] Mar 8, 2023
4742f5f
misc: review comments
4e6 Mar 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...uage-server/src/main/scala/org/enso/languageserver/boot/resource/RepoInitialization.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ class RepoInitialization(
} yield ()
initAction.onComplete {
case Success(()) =>
eventStream.publish(InitializedEvent.FileVersionsRepoInitialized)
eventStream.publish(InitializedEvent.VersionsRepoInitialized)
case Failure(ex) =>
logger.error(
"Failed to initialize SQL versions repo [{}]. {}",
Expand Down
10 changes: 5 additions & 5 deletions engine/language-server/src/main/scala/org/enso/languageserver/event/InitializedEvent.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ sealed trait InitializedEvent extends Event

object InitializedEvent {

case object SuggestionsRepoInitialized extends InitializedEvent
case object FileVersionsRepoInitialized extends InitializedEvent
case object TruffleContextInitialized extends InitializedEvent
case object InitializationFinished extends InitializedEvent
case object InitializationFailed extends InitializedEvent
case object SuggestionsRepoInitialized extends InitializedEvent
case object VersionsRepoInitialized extends InitializedEvent
case object TruffleContextInitialized extends InitializedEvent
case object InitializationFinished extends InitializedEvent
case object InitializationFailed extends InitializedEvent
}
62 changes: 15 additions & 47 deletions ...ne/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ final class SuggestionsHandler(
override def receive: Receive =
initializing(SuggestionsHandler.Initialization())

def initializing(init: SuggestionsHandler.Initialization): Receive = {
private def initializing(init: SuggestionsHandler.Initialization): Receive = {
case ProjectNameChangedEvent(oldName, newName) =>
logger.info(
"Initializing: project name changed from [{}] to [{}].",
Expand Down Expand Up @@ -178,35 +178,7 @@ final class SuggestionsHandler(
case _ => stash()
}

def verifying(
projectName: String,
graph: TypeGraph
): Receive = {
case Api.Response(_, Api.VerifyModulesIndexResponse(toRemove)) =>
logger.info("Verifying: got verification response.")
val removeAction = for {
_ <- versionsRepo.remove(toRemove)
_ <- suggestionsRepo.removeModules(toRemove)
} yield SuggestionsHandler.Verified
removeAction.pipeTo(self)

case SuggestionsHandler.Verified =>
logger.info("Verified.")
context.become(initialized(projectName, graph, Set(), State()))
unstashAll()

case Status.Failure(ex) =>
logger.error(
"Database verification failure [{}]. {}",
ex.getClass,
ex.getMessage
)

case _ =>
stash()
}

def initialized(
private def initialized(
projectName: String,
graph: TypeGraph,
clients: Set[ClientId],
Expand Down Expand Up @@ -235,7 +207,7 @@ final class SuggestionsHandler(
state.suggestionUpdatesQueue.enqueue(msg)

case SuggestionUpdatesBatch(updates) if state.isSuggestionUpdatesRunning =>
state.suggestionUpdatesQueue.enqueueAll(updates)
state.suggestionUpdatesQueue.prependAll(updates)

case SuggestionUpdatesBatch(updates) =>
val modules = updates.map(_.module)
Expand Down Expand Up @@ -493,6 +465,13 @@ final class SuggestionsHandler(
updates.foreach(sessionRouter ! _)
context.become(initialized(name, graph, clients, state))

case libraryLoaded: Api.LibraryLoaded =>
logger.debug(
Copy link
Member

@JaroslavTulach JaroslavTulach Feb 21, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@4e6's idea is to deliver the cached suggestion.json here. However I haven't hit a breakpoint placed here yet. I am not sure who's supposed to deliver this message? Probably we are missing a call to NotificationHandler.addListener that would register SuggestionHandler. Probably the listener shall be attached when case InitializedEvent.TruffleContextInitialized is delivered - by where can I find the EnsoContext at that moment?

Copy link
Member

@JaroslavTulach JaroslavTulach Feb 24, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One has to subscribe to Akka event bus to get the notification - f480e03

But now there is another problem: The SuggestionSerializationManager is in runtime, while this code is in language-server - how shall these two talk to each other? I believe this "library added" notification is send from following stacktrace:

at org.enso.interpreter.instrument.Endpoint.sendToClient(Handler.scala:48)
at org.enso.interpreter.instrument.NotificationHandler$InteractiveMode.sendMessage(NotificationHandler.scala:110)
at org.enso.interpreter.instrument.NotificationHandler$InteractiveMode.addedLibrary(NotificationHandler.scala:123)
at org.enso.interpreter.instrument.NotificationHandler$Forwarder.$anonfun$addedLibrary$1(NotificationHandler.scala:80)
at org.enso.interpreter.instrument.NotificationHandler$Forwarder.$anonfun$addedLibrary$1$adapted(NotificationHandler.scala:79)
at scala.collection.immutable.List.foreach(List.scala:333)
at org.enso.interpreter.instrument.NotificationHandler$Forwarder.addedLibrary(NotificationHandler.scala:79)
at org.enso.compiler.PackageRepository$Default.registerPackageInternal(PackageRepository.scala:326)

I believe the next step is to modify code in PackageRepository to check whether a suggestion cache exists for the library. If it does, read it and send the content of the suggestions via the Api.LibraryLoaded messages to the "language-server" project that can distribute it further. The following code shall be modified to send also librarySuggestions:

if (isLibrary) {
  val root = Path.of(pkg.root.toString)
  notificationHandler.addedLibrary(libraryName, libraryVersion, root) 
}

"Loaded Library [{}.{}]",
libraryLoaded.namespace,
libraryLoaded.name
)

case SuggestionUpdatesCompleted =>
if (state.suggestionUpdatesQueue.nonEmpty) {
self ! SuggestionUpdatesBatch(state.suggestionUpdatesQueue.removeAll())
Expand All @@ -515,17 +494,9 @@ final class SuggestionsHandler(
private def tryInitialize(state: SuggestionsHandler.Initialization): Unit = {
logger.debug("Trying to initialize with state [{}]", state)
state.initialized.fold(context.become(initializing(state))) {
case (name, graph) =>
case (projectName, graph) =>
logger.debug("Initialized with state [{}].", state)
val requestId = UUID.randomUUID()
suggestionsRepo.getAllModules
.map { modules =>
runtimeConnector ! Api.Request(
requestId,
Api.VerifyModulesIndexRequest(modules)
)
}
context.become(verifying(name, graph))
context.become(initialized(projectName, graph, Set(), State()))
unstashAll()
}
}
Expand Down Expand Up @@ -783,9 +754,6 @@ object SuggestionsHandler {
new ProjectNameUpdated(projectName, Seq())
}

/** The notification that the suggestions database has been verified. */
case object Verified

/** The notification that the suggestion updates are processed. */
case object SuggestionUpdatesCompleted

Expand Down Expand Up @@ -843,7 +811,7 @@ object SuggestionsHandler {
* @param config the server configuration
* @param contentRootManager the content root manager
* @param suggestionsRepo the suggestions repo
* @param fileVersionsRepo the file versions repo
* @param versionsRepo the versions repo
* @param sessionRouter the session router
* @param runtimeConnector the runtime connector
* @param docSectionsBuilder the builder of documentation sections
Expand All @@ -852,7 +820,7 @@ object SuggestionsHandler {
config: Config,
contentRootManager: ContentRootManager,
suggestionsRepo: SuggestionsRepo[Future],
fileVersionsRepo: VersionsRepo[Future],
versionsRepo: VersionsRepo[Future],
sessionRouter: ActorRef,
runtimeConnector: ActorRef,
docSectionsBuilder: DocSectionsBuilder = DocSectionsBuilder()
Expand All @@ -862,7 +830,7 @@ object SuggestionsHandler {
config,
contentRootManager,
suggestionsRepo,
fileVersionsRepo,
versionsRepo,
sessionRouter,
runtimeConnector,
docSectionsBuilder
Expand Down
5 changes: 3 additions & 2 deletions engine/language-server/src/main/scala/org/enso/languageserver/text/BufferRegistry.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,13 +102,14 @@ class BufferRegistry(
override def preStart(): Unit = {
logger.info("Starting initialization.")
context.system.eventStream
.subscribe(self, InitializedEvent.FileVersionsRepoInitialized.getClass)
.subscribe(self, InitializedEvent.
VersionsRepoInitialized.getClass)
}

override def receive: Receive = initializing

private def initializing: Receive = {
case InitializedEvent.FileVersionsRepoInitialized =>
case InitializedEvent.VersionsRepoInitialized =>
logger.info("Initiaized.")
context.become(running(Map.empty))
unstashAll()
Expand Down
10 changes: 5 additions & 5 deletions ...-server/src/test/scala/org/enso/languageserver/boot/resource/RepoInitializationSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ class RepoInitializationSpec

expectMsgAllOf(
InitializedEvent.SuggestionsRepoInitialized,
InitializedEvent.FileVersionsRepoInitialized
InitializedEvent.VersionsRepoInitialized
)
}

Expand Down Expand Up @@ -96,7 +96,7 @@ class RepoInitializationSpec

expectMsgAllOf(
InitializedEvent.SuggestionsRepoInitialized,
InitializedEvent.FileVersionsRepoInitialized
InitializedEvent.VersionsRepoInitialized
)
}

Expand All @@ -123,7 +123,7 @@ class RepoInitializationSpec
version1 shouldEqual SchemaVersion.CurrentVersion
expectMsgAllOf(
InitializedEvent.SuggestionsRepoInitialized,
InitializedEvent.FileVersionsRepoInitialized
InitializedEvent.VersionsRepoInitialized
)

// remove schema and re-initialize
Expand All @@ -138,7 +138,7 @@ class RepoInitializationSpec
version2 shouldEqual SchemaVersion.CurrentVersion
expectMsgAllOf(
InitializedEvent.SuggestionsRepoInitialized,
InitializedEvent.FileVersionsRepoInitialized
InitializedEvent.VersionsRepoInitialized
)
}

Expand Down Expand Up @@ -199,7 +199,7 @@ class RepoInitializationSpec
version2 shouldEqual SchemaVersion.CurrentVersion
expectMsgAllOf(
InitializedEvent.SuggestionsRepoInitialized,
InitializedEvent.FileVersionsRepoInitialized
InitializedEvent.VersionsRepoInitialized
)
}
}
Expand Down
Loading