From 00843249e94b240c0a0615daa7a8342f0ef536b7 Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Sat, 3 Jun 2023 01:08:50 +0800 Subject: [PATCH] nixd/fork: completion regression --- lib/lspserver/include/lspserver/Connection.h | 3 +- lib/lspserver/include/lspserver/LSPServer.h | 36 ++++--- lib/lspserver/src/Connection.cpp | 5 +- lib/nixd/include/nixd/Diagnostic.h | 2 + lib/nixd/include/nixd/EvalDraftStore.h | 36 +------ lib/nixd/include/nixd/Server.h | 49 +++++++-- lib/nixd/src/Diagnostic.cpp | 5 +- lib/nixd/src/ServerController.cpp | 84 ++++++++++++++- lib/nixd/src/ServerSerialization.cpp | 28 ++++- lib/nixd/src/ServerWorker.cpp | 104 +++++++++---------- tools/nixd/nixd.cpp | 3 +- tools/nixd/test/completion-attrset.md | 2 +- tools/nixd/test/completion.md | 2 +- 13 files changed, 238 insertions(+), 121 deletions(-) diff --git a/lib/lspserver/include/lspserver/Connection.h b/lib/lspserver/include/lspserver/Connection.h index 2f292e6d7..99e911d9e 100644 --- a/lib/lspserver/include/lspserver/Connection.h +++ b/lib/lspserver/include/lspserver/Connection.h @@ -31,7 +31,7 @@ class MessageHandler { }; class InboundPort { -private: +public: int In; JSONStreamStyle StreamStyle = JSONStreamStyle::Standard; @@ -40,7 +40,6 @@ class InboundPort { bool readDelimitedMessage(std::string &JSONString); -public: InboundPort(int In = STDIN_FILENO, JSONStreamStyle StreamStyle = JSONStreamStyle::Standard) : In(In), StreamStyle(StreamStyle){}; diff --git a/lib/lspserver/include/lspserver/LSPServer.h b/lib/lspserver/include/lspserver/LSPServer.h index 1e09dcd72..18be67580 100644 --- a/lib/lspserver/include/lspserver/LSPServer.h +++ b/lib/lspserver/include/lspserver/LSPServer.h @@ -48,35 +48,41 @@ class LSPServer : public MessageHandler { int bindReply(Callback); void callMethod(llvm::StringRef Method, llvm::json::Value Params, - Callback CB) { + Callback CB, OutboundPort *O) { llvm::json::Value ID(bindReply(std::move(CB))); log("--> call {0}({1})", Method, ID.getAsInteger()); - Out->call(Method, Params, ID); + O->call(Method, Params, ID); } protected: HandlerRegistry Registry; template llvm::unique_function - mkOutNotifiction(llvm::StringRef Method) { - return [Method = Method, this](const T &Params) { + mkOutNotifiction(llvm::StringRef Method, OutboundPort *O = nullptr) { + if (!O) + O = Out.get(); + return [=, this](const T &Params) { log("--> notify {0}", Method); - Out->notify(Method, Params); + O->notify(Method, Params); }; } template llvm::unique_function)> - mkOutMethod(llvm::StringRef Method) { + mkOutMethod(llvm::StringRef Method, OutboundPort *O = nullptr) { + if (!O) + O = Out.get(); return [=, this](const ParamTy &Params, Callback Reply) { - callMethod(Method, Params, - [=, Reply = std::move(Reply)]( - llvm::Expected Response) mutable { - if (!Response) - return Reply(Response.takeError()); - Reply(parseParam(std::move(*Response), Method, - "reply")); - }); + callMethod( + Method, Params, + [=, Reply = std::move(Reply)]( + llvm::Expected Response) mutable { + if (!Response) + return Reply(Response.takeError()); + Reply( + parseParam(std::move(*Response), Method, "reply")); + }, + O); }; } @@ -84,6 +90,8 @@ class LSPServer : public MessageHandler { LSPServer(std::unique_ptr In, std::unique_ptr Out) : In(std::move(In)), Out(std::move(Out)){}; void run(); + + void switchStreamStyle(JSONStreamStyle Style) { In->StreamStyle = Style; } }; } // namespace lspserver diff --git a/lib/lspserver/src/Connection.cpp b/lib/lspserver/src/Connection.cpp index 4327c3fe5..3018665fa 100644 --- a/lib/lspserver/src/Connection.cpp +++ b/lib/lspserver/src/Connection.cpp @@ -140,14 +140,13 @@ bool InboundPort::readStandardMessage(std::string &JSONString) { JSONString.resize(ContentLength); for (size_t Pos = 0, Read; Pos < ContentLength; Pos += Read) { - Read = read(In, &JSONString[Pos], ContentLength - Pos); + Read = read(In, JSONString.data() + Pos, ContentLength - Pos); if (Read == 0) { elog("Input was aborted. Read only {0} bytes of expected {1}.", Pos, ContentLength); return false; } - Pos += Read; } return true; } @@ -200,6 +199,8 @@ void InboundPort::loop(MessageHandler &Handler) { if (!dispatch(*ExpectedParsedJSON, Handler)) return; } else { + auto Err = ExpectedParsedJSON.takeError(); + elog("The received json cannot be parsed, reason: {0}", Err); return; } } else { diff --git a/lib/nixd/include/nixd/Diagnostic.h b/lib/nixd/include/nixd/Diagnostic.h index 6c684a4d1..2dd7e21aa 100644 --- a/lib/nixd/include/nixd/Diagnostic.h +++ b/lib/nixd/include/nixd/Diagnostic.h @@ -5,6 +5,8 @@ namespace nixd { +std::string stripANSI(std::string Msg); + std::vector mkDiagnostics(const nix::BaseError &); lspserver::Position translatePosition(const nix::AbstractPos &P); diff --git a/lib/nixd/include/nixd/EvalDraftStore.h b/lib/nixd/include/nixd/EvalDraftStore.h index 8843b4255..ecd8ce7c9 100644 --- a/lib/nixd/include/nixd/EvalDraftStore.h +++ b/lib/nixd/include/nixd/EvalDraftStore.h @@ -93,40 +93,8 @@ struct IValueEvalResult { EvalASTForest Forest; std::unique_ptr Session; - static const IValueEvalResult * - search(const std::string &Path, - const std::vector &Order) { - for (const auto &Result : Order) { - try { - Result->Forest.at(Path); - return Result; - } catch (...) { - } - } - return nullptr; - } + IValueEvalResult(decltype(Forest) Forest, decltype(Session) Session) + : Forest(std::move(Forest)), Session(std::move(Session)) {} }; -/// Ownes `EvalASTForest`s, mark it is evaluated or not, clients query on this -/// cache to find a suitable AST Tree -struct ForestCache { - - IValueEvalResult EvaluatedResult; - - IValueEvalResult NonEmptyResult; - - enum class ASTPreference { Evaluated, NonEmpty }; - - [[nodiscard]] const IValueEvalResult * - searchAST(const std::string &Path, ASTPreference Preference) const { - switch (Preference) { - case ASTPreference::Evaluated: - return IValueEvalResult::search(Path, - {&EvaluatedResult, &NonEmptyResult}); - case ASTPreference::NonEmpty: - return IValueEvalResult::search(Path, - {&NonEmptyResult, &EvaluatedResult}); - } - }; -}; } // namespace nixd diff --git a/lib/nixd/include/nixd/Server.h b/lib/nixd/include/nixd/Server.h index 45294c516..d13d3b971 100644 --- a/lib/nixd/include/nixd/Server.h +++ b/lib/nixd/include/nixd/Server.h @@ -13,10 +13,29 @@ #include #include +#include #include +#include -namespace nixd { +// Extension to `lspserver` +namespace lspserver { + +llvm::json::Value toJSON(const CompletionContext &R); + +llvm::json::Value toJSON(const TextDocumentPositionParams &R); + +llvm::json::Value toJSON(const CompletionParams &R); + +bool fromJSON(const llvm::json::Value &Params, CompletionItem &R, + llvm::json::Path P); + +bool fromJSON(const llvm::json::Value &Params, CompletionList &R, + llvm::json::Path P); + +} // namespace lspserver + +namespace nixd { // namespace nixd namespace configuration { @@ -75,6 +94,10 @@ bool fromJSON(const llvm::json::Value &, lspserver::PublishDiagnosticsParams &, bool fromJSON(const llvm::json::Value &, Diagnostics &, llvm::json::Path); llvm::json::Value toJSON(const Diagnostics &); +struct Completion : WorkerMessage { + lspserver::CompletionList List; +}; + } // namespace ipc /// The server instance, nix-related language features goes here @@ -94,15 +117,20 @@ class Server : public lspserver::LSPServer { struct Proc { std::unique_ptr ToPipe; std::unique_ptr FromPipe; + std::unique_ptr OutPort; + std::unique_ptr OwnedStream; + nix::Pid Pid; WorkspaceVersionTy WorkspaceVersion; std::thread InputDispatcher; - Proc(decltype(ToPipe) ToPipe, decltype(FromPipe) FromPipe, pid_t Pid, - decltype(WorkspaceVersion) WorkspaceVersion, + Proc(decltype(ToPipe) ToPipe, decltype(FromPipe) FromPipe, + decltype(OutPort) OutPort, decltype(OwnedStream) OwnedStream, + pid_t Pid, decltype(WorkspaceVersion) WorkspaceVersion, decltype(InputDispatcher) InputDispatcher) - : ToPipe(std::move(ToPipe)), FromPipe(std::move(FromPipe)), Pid(Pid), - WorkspaceVersion(WorkspaceVersion), + : ToPipe(std::move(ToPipe)), FromPipe(std::move(FromPipe)), + OutPort(std::move(OutPort)), OwnedStream(std::move(OwnedStream)), + Pid(Pid), WorkspaceVersion(WorkspaceVersion), InputDispatcher(std::move(InputDispatcher)) {} [[nodiscard]] nix::AutoCloseFD to() const { @@ -155,7 +183,8 @@ class Server : public lspserver::LSPServer { //---------------------------------------------------------------------------/ // Worker members llvm::unique_function WorkerDiagnostic; - ForestCache FCache; + + std::unique_ptr IER; public: Server(std::unique_ptr In, @@ -199,11 +228,17 @@ class Server : public lspserver::LSPServer { const lspserver::DidChangeConfigurationParams &) { fetchConfig(); } + void onHover(const lspserver::TextDocumentPositionParams &, lspserver::Callback); + void onWorkerHover(const lspserver::TextDocumentPositionParams &, + lspserver::Callback); void onCompletion(const lspserver::CompletionParams &, - lspserver::Callback); + lspserver::Callback); + + void onWorkerCompletion(const lspserver::CompletionParams &, + lspserver::Callback); }; }; // namespace nixd diff --git a/lib/nixd/src/Diagnostic.cpp b/lib/nixd/src/Diagnostic.cpp index f0ce15b11..d64dde2ce 100644 --- a/lib/nixd/src/Diagnostic.cpp +++ b/lib/nixd/src/Diagnostic.cpp @@ -15,8 +15,9 @@ FUNC(ANSI_BLUE) \ FUNC(ANSI_MAGENTA) \ FUNC(ANSI_CYAN) +namespace nixd { -static std::string stripANSI(std::string Msg){ +std::string stripANSI(std::string Msg) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpedantic" #define REMOVE_ANSI_STR_FUNC(ANSI_STR) \ @@ -32,8 +33,6 @@ static std::string stripANSI(std::string Msg){ return Msg; } -namespace nixd { - std::vector mkDiagnostics(const nix::BaseError &Err) { std::vector Ret; auto ErrPos = Err.info().errPos; diff --git a/lib/nixd/src/ServerController.cpp b/lib/nixd/src/ServerController.cpp index 41edeab80..76ec6d435 100644 --- a/lib/nixd/src/ServerController.cpp +++ b/lib/nixd/src/ServerController.cpp @@ -84,9 +84,17 @@ void Server::updateWorkspaceVersion() { IPort->loop(*this); }); WorkerInputDispatcher.detach(); - auto WorkerProc = std::make_unique(std::move(To), std::move(From), - ForkPID, WorkspaceVersion, - std::move(WorkerInputDispatcher)); + + auto ProcFdStream = + std::make_unique(To->writeSide.get(), false); + + auto OutPort = + std::make_unique(*ProcFdStream, false); + + auto WorkerProc = std::make_unique( + std::move(To), std::move(From), std::move(OutPort), + std::move(ProcFdStream), ForkPID, WorkspaceVersion, + std::move(WorkerInputDispatcher)); Workers.emplace_back(std::move(WorkerProc)); if (Workers.size() > 5) { @@ -183,6 +191,9 @@ Server::Server(std::unique_ptr In, /// IPC Registry.addNotification("nixd/worker/diagnostic", this, &Server::onWorkerDiagnostic); + + Registry.addMethod("nixd/worker/textDocument/completion", this, + &Server::onWorkerCompletion); } void Server::onDocumentDidOpen( @@ -256,4 +267,71 @@ void Server::onWorkerDiagnostic(const ipc::Diagnostics &Diag) { } } +//-----------------------------------------------------------------------------/ +// Completion + +void Server::onCompletion( + const lspserver::CompletionParams &Params, + lspserver::Callback Reply) { + auto Thread = std::thread([=, Reply = std::move(Reply), this]() mutable { + // For all active workers, send the completion request + + std::vector ListStore(Workers.size()); + + auto *StorePtr = &ListStore; + + size_t I = 0; + for (const auto &Worker : Workers) { + auto ComplectionRequest = + mkOutMethod( + "nixd/worker/textDocument/completion", Worker->OutPort.get()); + + ComplectionRequest( + Params, + [I, &StorePtr](llvm::Expected Result) { + // The worker answered our request, fill the completion + // lists then. + if (Result) { + lspserver::log( + "received result from our client, which has {0} item(s)", + Result.get().items.size()); + if (StorePtr) { + (*StorePtr)[I] = Result.get(); + } else if (!Result.get().items.empty()) { + lspserver::elog( + "ignored non-empty response, because it's to late."); + } + } + }); + I++; + } + // Wait for our client, this is currently hardcoded + usleep(5e5); + + // Reset the store ptr in event handling module, so that client reply after + // 'usleep' will not write the store (to avoid data race) + StorePtr = nullptr; + + // brute-force iterating over the result, and choose the biggest item + size_t BestIdx = 0; + size_t BestSize = 0; + for (size_t I = 0; I < ListStore.size(); I++) { + auto LSize = ListStore[I].items.size(); + if (LSize >= BestSize) { + BestIdx = I; + BestSize = LSize; + } + } + // And finally, reply our client + lspserver::log("chosed {0} completion lists.", BestSize); + Reply(ListStore.at(BestIdx)); + ListStore.resize(0); + }); + + Thread.detach(); +} + +void Server::onHover(const lspserver::TextDocumentPositionParams &, + lspserver::Callback) {} + } // namespace nixd diff --git a/lib/nixd/src/ServerSerialization.cpp b/lib/nixd/src/ServerSerialization.cpp index f121df82a..271d1a1fc 100644 --- a/lib/nixd/src/ServerSerialization.cpp +++ b/lib/nixd/src/ServerSerialization.cpp @@ -49,13 +49,13 @@ Value toJSON(const Diagnostics &R) { Base.getAsObject()->insert({"Params", R.Params}); return Base; } - } // namespace ipc } // namespace nixd namespace lspserver { +using llvm::json::Object; using llvm::json::ObjectMapper; using llvm::json::Value; @@ -73,4 +73,30 @@ bool fromJSON(const Value &Params, PublishDiagnosticsParams &R, O.map("diagnostics", R.diagnostics); } +Value toJSON(const CompletionContext &R) { + return Object{{"triggerCharacter", R.triggerCharacter}, + {"triggerKind", static_cast(R.triggerKind)}}; +} + +Value toJSON(const TextDocumentPositionParams &R) { + return Object{{"textDocument", R.textDocument}, {"position", R.position}}; +} + +Value toJSON(const CompletionParams &R) { + Value Base = toJSON(static_cast(R)); + Base.getAsObject()->insert({"context", R.context}); + Base.getAsObject()->insert({"limit", R.limit}); + return Base; +} + +bool fromJSON(const Value &Params, CompletionItem &R, llvm::json::Path P) { + ObjectMapper O(Params, P); + return O && O.map("label", R.label) && O.map("kind", R.kind); +} + +bool fromJSON(const Value &Params, CompletionList &R, llvm::json::Path P) { + ObjectMapper O(Params, P); + return O && O.map("isIncomplete", R.isIncomplete) && O.map("items", R.items); +} + } // namespace lspserver diff --git a/lib/nixd/src/ServerWorker.cpp b/lib/nixd/src/ServerWorker.cpp index 899070f1f..c9ff7ade4 100644 --- a/lib/nixd/src/ServerWorker.cpp +++ b/lib/nixd/src/ServerWorker.cpp @@ -1,3 +1,4 @@ +#include "lspserver/Connection.h" #include "nixd/Diagnostic.h" #include "nixd/Server.h" @@ -70,6 +71,14 @@ void Server::switchToEvaluator() { nix::initPlugins(); nix::initGC(); + /// Basically communicate with the controller in standard mode. because we do + /// not support "lit-test" outbound port. + switchStreamStyle(lspserver::JSONStreamStyle::Standard); + + // unregister "Procs" in worker process, they are managed by controller + for (auto &Worker : Workers) + Worker->Pid.release(); + WorkerDiagnostic = mkOutNotifiction("nixd/worker/diagnostic"); @@ -110,22 +119,21 @@ void Server::eval(const std::string &Fallback) { WorkerDiagnostic(Diagnostics); - if (!ILR.InjectionErrors.empty()) { - if (!ILR.Forest.empty()) - FCache.NonEmptyResult = {std::move(ILR.Forest), std::move(Session)}; - } else { - const std::string EvalationErrorFile = "/evaluation-error-file.nix"; - try { - Session->eval(Installable); - FCache.EvaluatedResult = {std::move(ILR.Forest), std::move(Session)}; - } catch (nix::BaseError &BE) { - } catch (...) { - } + try { + Session->eval(Installable); + lspserver::log("evaluation done on worspace version: {0}", + WorkspaceVersion); + } catch (nix::BaseError &BE) { + lspserver::elog("evaluation error on workspace version: {0}, reason: {1}", + WorkspaceVersion, stripANSI(BE.what())); + } catch (...) { } + IER = std::make_unique(std::move(ILR.Forest), + std::move(Session)); } -void Server::onCompletion(const lspserver::CompletionParams &Params, - lspserver::Callback Reply) { +void Server::onWorkerCompletion(const lspserver::CompletionParams &Params, + lspserver::Callback Reply) { using namespace lspserver; std::string CompletionFile = Params.textDocument.uri.file().str(); @@ -136,60 +144,52 @@ void Server::onCompletion(const lspserver::CompletionParams &Params, ~ReplyRAII() { R(Response); }; } RR(std::move(Reply)); - const auto *IER = - FCache.searchAST(CompletionFile, ForestCache::ASTPreference::Evaluated); - - if (!IER) - return; - - auto AST = IER->Forest.at(CompletionFile); + auto GetStaticEnvItems = [this]() { + return CompletionHelper::fromStaticEnv( + IER->Session->getState()->symbols, + *IER->Session->getState()->staticBaseEnv); + }; - auto State = IER->Session->getState(); - // Lookup an AST node, and get it's 'Env' after evaluation - CompletionHelper::Items Items; - lspserver::vlog("current trigger character is {0}", - Params.context.triggerCharacter); + RR.Response.isIncomplete = false; - if (Params.context.triggerCharacter == ".") { - try { - auto *Node = AST->lookupPosition(Params.position); - auto Value = AST->getValue(Node); - if (Value.type() == nix::ValueType::nAttrs) { - // Traverse attribute bindings - for (auto Binding : *Value.attrs) { - Items.emplace_back( - CompletionItem{.label = State->symbols[Binding.name], - .kind = CompletionItemKind::Field}); + try { + auto AST = IER->Forest.at(CompletionFile); + auto State = IER->Session->getState(); + // Lookup an AST node, and get it's 'Env' after evaluation + CompletionHelper::Items Items; + lspserver::log("current trigger character is {0}", + Params.context.triggerCharacter); + + if (Params.context.triggerCharacter == ".") { + try { + auto *Node = AST->lookupPosition(Params.position); + auto Value = AST->getValue(Node); + if (Value.type() == nix::ValueType::nAttrs) { + // Traverse attribute bindings + for (auto Binding : *Value.attrs) { + Items.emplace_back( + CompletionItem{.label = State->symbols[Binding.name], + .kind = CompletionItemKind::Field}); + } } + } catch (...) { } - } catch (...) { - } - } else { - try { + } else { auto *Node = AST->lookupPosition(Params.position); const auto *ExprEnv = AST->getEnv(Node); Items = CompletionHelper::fromEnvRecursive( State->symbols, *State->staticBaseEnv, *ExprEnv); - } catch (const std::out_of_range &) { - // Fallback to staticEnv only - Items = CompletionHelper::fromStaticEnv(State->symbols, - *State->staticBaseEnv); } + RR.Response.items = Items; + } catch (std::out_of_range &) { } - RR.Response.isIncomplete = false; - RR.Response.items = Items; } -void Server::onHover(const lspserver::TextDocumentPositionParams &Paras, - lspserver::Callback Reply) { +void Server::onWorkerHover(const lspserver::TextDocumentPositionParams &Paras, + lspserver::Callback Reply) { using namespace lspserver; std::string HoverFile = Paras.textDocument.uri.file().str(); - const auto *IER = - FCache.searchAST(HoverFile, ForestCache::ASTPreference::NonEmpty); - - if (!IER) - return; auto AST = IER->Forest.at(HoverFile); diff --git a/tools/nixd/nixd.cpp b/tools/nixd/nixd.cpp index fd72233c6..307b0cc52 100644 --- a/tools/nixd/nixd.cpp +++ b/tools/nixd/nixd.cpp @@ -63,7 +63,8 @@ int main(int argc, char *argv[]) { InputStyle = JSONStreamStyle::Delimited; LogLevel = Logger::Level::Verbose; PrettyPrint = true; - WaitWorker = 2e5; // 0.2s + if (!WaitWorker) + WaitWorker = 2e5; // 0.2s } StreamLogger Logger(llvm::errs(), LogLevel); diff --git a/tools/nixd/test/completion-attrset.md b/tools/nixd/test/completion-attrset.md index ba15b80a3..996d84ed5 100644 --- a/tools/nixd/test/completion-attrset.md +++ b/tools/nixd/test/completion-attrset.md @@ -1,4 +1,4 @@ -# RUN: nixd --lit-test < %s | FileCheck %s +# RUN: nixd --lit-test --wait-worker=1000000 < %s | FileCheck %s <-- initialize(0) diff --git a/tools/nixd/test/completion.md b/tools/nixd/test/completion.md index e1d37d8cd..a95e9a96a 100644 --- a/tools/nixd/test/completion.md +++ b/tools/nixd/test/completion.md @@ -1,4 +1,4 @@ -# RUN: nixd --lit-test < %s | FileCheck %s +# RUN: nixd -wait-worker 1000000 --lit-test < %s | FileCheck %s <-- initialize(0)