Multi-IDE Features for 2.9

sdk/compiler/daml-extension/src/extension.ts

@@ -311,7 +311,7 @@ export function createLanguageClient(
     // The user has not made an explicit choice.
     args.push("--telemetry-ignored");
   }
-  if (multiIDEVerbose === true) {
+  if (multiIDEVerbose === true && multiIDESupport === true) {
     args.push("--verbose=yes");
   }
   const extraArgsString = config.get("extraArguments", "").trim();

sdk/compiler/damlc/lib/DA/Cli/Damlc/Command/MultiIde/Parsing.hs

@@ -17,6 +17,7 @@ module DA.Cli.Damlc.Command.MultiIde.Parsing (
   putReqMethodAll,
   putReqMethodSingleFromClient,
   putReqMethodSingleFromServer,
+  putReqMethodSingleFromServerCoordinator,
   putServerReq,
 ) where
 
@@ -68,7 +69,12 @@ putChunk handle payload = do
 putReqMethodSingleFromServer
   :: forall (m :: LSP.Method 'LSP.FromServer 'LSP.Request)
   .  MethodTrackerVar 'LSP.FromServer -> FilePath -> LSP.LspId m -> LSP.SMethod m -> IO ()
-putReqMethodSingleFromServer tracker home id method = putReqMethod tracker id $ TrackedSingleMethodFromServer method home
+putReqMethodSingleFromServer tracker home id method = putReqMethod tracker id $ TrackedSingleMethodFromServer method $ Just home
+
+putReqMethodSingleFromServerCoordinator
+  :: forall (m :: LSP.Method 'LSP.FromServer 'LSP.Request)
+  .  MethodTrackerVar 'LSP.FromServer -> LSP.LspId m -> LSP.SMethod m -> IO ()
+putReqMethodSingleFromServerCoordinator tracker id method = putReqMethod tracker id $ TrackedSingleMethodFromServer method Nothing
 
 putReqMethodSingleFromClient
   :: forall (m :: LSP.Method 'LSP.FromClient 'LSP.Request)
@@ -149,12 +155,12 @@ parseServerMessageWithTracker tracker selfIde val = pickReqMethodTo tracker $ \e
 parseClientMessageWithTracker
   :: MethodTrackerVar 'LSP.FromServer
   -> Aeson.Value
-  -> IO (Either String (LSP.FromClientMessage' (Product LSP.SMethod (Const FilePath))))
+  -> IO (Either String (LSP.FromClientMessage' (Product LSP.SMethod (Const (Maybe FilePath)))))
 parseClientMessageWithTracker tracker val = pickReqMethodTo tracker $ \extract ->
   case Aeson.parseEither (LSP.parseClientMessage (wrapParseMessageLookup . extract)) val of
     Right (LSP.FromClientMess meth mess) -> (Right (LSP.FromClientMess meth mess), Nothing)
-    Right (LSP.FromClientRsp (Pair (TrackedSingleMethodFromServer method home) (Const newIxMap)) rsp) ->
-      (Right (LSP.FromClientRsp (Pair method (Const home)) rsp), Just newIxMap)
+    Right (LSP.FromClientRsp (Pair (TrackedSingleMethodFromServer method mHome) (Const newIxMap)) rsp) ->
+      (Right (LSP.FromClientRsp (Pair method (Const mHome)) rsp), Just newIxMap)
     Left msg -> (Left msg, Nothing)
 
 -- Takes a message from server and stores it if its a request, so that later messages from the client can deduce response context

sdk/compiler/damlc/lib/DA/Cli/Damlc/Command/MultiIde/Prefixing.hs

@@ -206,7 +206,7 @@ stripLspPrefix (LSP.IdString (T.uncons -> Just ('t', rest))) = LSP.IdString $ T.
 stripLspPrefix t = t
 
 -- Prefixes applied to builtin and custom requests. Notifications do not have ids, responses do not need this logic.
-addLspPrefixToServerMessage :: SubIDE -> LSP.FromServerMessage -> LSP.FromServerMessage
+addLspPrefixToServerMessage :: SubIDEInstance -> LSP.FromServerMessage -> LSP.FromServerMessage
 addLspPrefixToServerMessage _ res@(LSP.FromServerRsp _ _) = res
 addLspPrefixToServerMessage ide res@(LSP.FromServerMess method params) =
   case LSP.splitServerMethod method of

sdk/compiler/damlc/lib/DA/Cli/Damlc/Command/MultiIde/Types.hs

@@ -20,8 +20,11 @@ import Control.Concurrent.STM.TMVar
 import Control.Concurrent.MVar
 import Control.Monad.STM
 import qualified Data.ByteString.Lazy as BSL
+import Data.Function (on)
 import qualified Data.IxMap as IM
 import qualified Data.Map as Map
+import Data.Maybe (fromMaybe)
+import qualified Data.Set as Set
 import qualified Data.Text as T
 import qualified Language.LSP.Types as LSP
 import System.IO.Extra
@@ -35,7 +38,7 @@ data TrackedMethod (m :: LSP.Method from 'LSP.Request) where
   TrackedSingleMethodFromServer
     :: forall (m :: LSP.Method 'LSP.FromServer 'LSP.Request)
     .  LSP.SMethod m
-    -> FilePath -- Also store the IDE that sent the request
+    -> Maybe FilePath -- Also store the IDE that sent the request (or don't, for requests sent by the coordinator)
     -> TrackedMethod m
   TrackedAllMethod :: forall (m :: LSP.Method 'LSP.FromClient 'LSP.Request).
     { tamMethod :: LSP.SMethod m
@@ -59,7 +62,7 @@ tmMethod (TrackedAllMethod {tamMethod}) = tamMethod
 type MethodTracker (from :: LSP.From) = IM.IxMap @(LSP.Method from 'LSP.Request) LSP.LspId TrackedMethod
 type MethodTrackerVar (from :: LSP.From) = TVar (MethodTracker from)
 
-data SubIDE = SubIDE
+data SubIDEInstance = SubIDEInstance
   { ideInhandleAsync :: Async ()
   , ideInHandle :: Handle
   , ideInHandleChannel :: TChan BSL.ByteString
@@ -70,32 +73,60 @@ data SubIDE = SubIDE
   , ideMessageIdPrefix :: T.Text
     -- ^ Some unique string used to prefix message ids created by the SubIDE, to avoid collisions with other SubIDEs
     -- We use the stringified process ID
-  , ideActive :: Bool
   , ideUnitId :: String
     -- ^ Unit ID of the package this SubIDE handles
     -- Of the form "daml-script-0.0.1"
   }
 
+instance Eq SubIDEInstance where
+  -- ideMessageIdPrefix is derived from process id, so this equality is of the process.
+  (==) = (==) `on` ideMessageIdPrefix
+
+instance Ord SubIDEInstance where
+  -- ideMessageIdPrefix is derived from process id, so this ordering is of the process.
+  compare = compare `on` ideMessageIdPrefix
+
+-- We store an optional main ide, the currently closing ides (kept only so they can reply to their shutdowns), and open files
+-- open files must outlive the main subide so we can re-send the TextDocumentDidOpen messages on new ide startup
+data SubIDEData = SubIDEData
+  { ideDataMain :: Maybe SubIDEInstance
+  , ideDataClosing :: Set.Set SubIDEInstance
+  , ideDataOpenFiles :: Set.Set FilePath
+  }
+
+defaultSubIDEData :: SubIDEData
+defaultSubIDEData = SubIDEData Nothing Set.empty Set.empty
+
+lookupSubIde :: FilePath -> SubIDEs -> SubIDEData
+lookupSubIde home ides = fromMaybe defaultSubIDEData $ Map.lookup home ides
+
 -- SubIDEs placed in a TMVar. The emptyness representents a modification lock.
 -- The lock unsures the following properties:
 --   If multiple messages are sent to a new IDE at the same time, the first will create and hold a lock, while the rest wait on that lock (avoid multiple create)
 --   We never attempt to send messages on a stale IDE. If we ever read SubIDEsVar with the intent to send a message on a SubIDE, we must hold the so a shutdown
 --     cannot be sent on that IDE until we are done. This ensures that when a shutdown does occur, it is impossible for non-shutdown messages to be added to the
 --     queue after the shutdown.
-type SubIDEs = Map.Map FilePath SubIDE
+type SubIDEs = Map.Map FilePath SubIDEData
 type SubIDEsVar = TMVar SubIDEs
 
-onlyActiveSubIdes :: SubIDEs -> SubIDEs
-onlyActiveSubIdes = Map.filter ideActive
-
 -- Stores the initialize messages sent by the client to be forwarded to SubIDEs when they are created.
 type InitParams = LSP.InitializeParams
 type InitParamsVar = MVar InitParams
 
 -- Maps a packages unit id to its source location, using PackageOnDisk for all packages in multi-package.yaml
 -- and PackageInDar for all known dars (currently extracted from data-dependencies)
-data PackageSourceLocation = PackageOnDisk FilePath | PackageInDar FilePath
+data PackageSourceLocation = PackageOnDisk FilePath | PackageInDar FilePath deriving Show
 type MultiPackageYamlMapping = Map.Map String PackageSourceLocation
+type MultiPackageYamlMappingVar = TMVar MultiPackageYamlMapping
+
+-- Maps a dar path to the list of packages that directly depend on it
+type DarDependentPackages = Map.Map FilePath [FilePath]
+type DarDependentPackagesVar = TMVar DarDependentPackages
+
+-- "Cache" for the home path of files
+-- Cleared on daml.yaml modification and file deletion
+type SourceFileHomes = Map.Map FilePath FilePath
+type SourceFileHomesVar = TMVar SourceFileHomes
 
 data MultiIdeState = MultiIdeState
   { fromClientMethodTrackerVar :: MethodTrackerVar 'LSP.FromClient
@@ -105,19 +136,24 @@ data MultiIdeState = MultiIdeState
   , subIDEsVar :: SubIDEsVar
   , initParamsVar :: InitParamsVar
   , toClientChan :: TChan BSL.ByteString
-  , multiPackageMapping :: MultiPackageYamlMapping
+  , multiPackageMappingVar :: MultiPackageYamlMappingVar
+  , darDependentPackagesVar :: DarDependentPackagesVar
   , debugPrint :: String -> IO ()
   , multiPackageHome :: FilePath
   , defaultPackagePath :: FilePath
+  , sourceFileHomesVar :: SourceFileHomesVar
   }
 
-newMultiIdeState :: MultiPackageYamlMapping -> FilePath -> FilePath -> (String -> IO ()) -> IO MultiIdeState
-newMultiIdeState multiPackageMapping multiPackageHome defaultPackagePath debugPrint = do
+newMultiIdeState :: FilePath -> FilePath -> (String -> IO ()) -> IO MultiIdeState
+newMultiIdeState multiPackageHome defaultPackagePath debugPrint = do
   (fromClientMethodTrackerVar :: MethodTrackerVar 'LSP.FromClient) <- newTVarIO IM.emptyIxMap
   (fromServerMethodTrackerVar :: MethodTrackerVar 'LSP.FromServer) <- newTVarIO IM.emptyIxMap
   subIDEsVar <- newTMVarIO @SubIDEs mempty
   initParamsVar <- newEmptyMVar @InitParams
   toClientChan <- atomically newTChan
+  multiPackageMappingVar <- newTMVarIO @MultiPackageYamlMapping mempty
+  darDependentPackagesVar <- newTMVarIO @DarDependentPackages mempty
+  sourceFileHomesVar <- newTMVarIO @SourceFileHomes mempty
   pure MultiIdeState {..}
 
 -- Forwarding

sdk/compiler/damlc/lib/DA/Cli/Damlc/Command/MultiIde/Util.hs

@@ -26,7 +26,7 @@ import Data.Maybe (fromMaybe)
 import qualified Language.LSP.Types as LSP
 import qualified Language.LSP.Types.Capabilities as LSP
 import System.Directory (doesDirectoryExist, listDirectory, withCurrentDirectory, canonicalizePath)
-import System.FilePath (takeDirectory)
+import System.FilePath (takeDirectory, takeExtension)
 import System.IO.Extra
 import System.IO.Unsafe (unsafePerformIO)
 
@@ -153,6 +153,17 @@ initializeResult = LSP.InitializeResult
     true = Just (LSP.InL True)
     false = Just (LSP.InL False)
 
+registerFileWatchersMessage :: LSP.RequestMessage 'LSP.ClientRegisterCapability
+registerFileWatchersMessage =
+  LSP.RequestMessage "2.0" (LSP.IdString "MultiIdeWatchedFiles") LSP.SClientRegisterCapability $ LSP.RegistrationParams $ LSP.List
+    [ LSP.SomeRegistration $ LSP.Registration "MultiIdeWatchedFiles" LSP.SWorkspaceDidChangeWatchedFiles $ LSP.DidChangeWatchedFilesRegistrationOptions $ LSP.List
+      [ LSP.FileSystemWatcher "**/multi-package.yaml" Nothing
+      , LSP.FileSystemWatcher "**/daml.yaml" Nothing
+      , LSP.FileSystemWatcher "**/*.dar" Nothing
+      , LSP.FileSystemWatcher "**/*.daml" Nothing
+      ]
+    ]
+
 castLspId :: LSP.LspId m -> LSP.LspId m'
 castLspId (LSP.IdString s) = LSP.IdString s
 castLspId (LSP.IdInt i) = LSP.IdInt i
@@ -178,8 +189,10 @@ unitIdAndDepsFromDamlYaml :: FilePath -> IO (Either ConfigError (String, [FilePa
 unitIdAndDepsFromDamlYaml path = do
   handle (\(e :: ConfigError) -> return $ Left e) $ runExceptT $ do
     project <- lift $ readProjectConfig $ ProjectPath path
-    deps <- except $ fromMaybe [] <$> queryProjectConfig ["data-dependencies"] project
-    canonDeps <- lift $ withCurrentDirectory path $ traverse canonicalizePath deps
+    dataDeps <- except $ fromMaybe [] <$> queryProjectConfig ["data-dependencies"] project
+    directDeps <- except $ fromMaybe [] <$> queryProjectConfig ["dependencies"] project
+    let directDarDeps = filter (\dep -> takeExtension dep == ".dar") directDeps
+    canonDeps <- lift $ withCurrentDirectory path $ traverse canonicalizePath $ dataDeps <> directDarDeps
     name <- except $ queryProjectConfigRequired ["name"] project
     version <- except $ queryProjectConfigRequired ["version"] project
     pure (name <> "-" <> version, canonDeps)