Initialize ExportsMap using hiedb exports (#1989)

* Initialize ExportsMap using hiedb exports

* bump version number

* Add temporary source package to ghc901 cabal project as well

* fix stack descriptors

* remove source package overrides

* refresh cabal index state

* fix 9.0.1 build

cabal-ghc901.project

@@ -41,7 +41,6 @@ source-repository-package
   tag: b6245884ae83e00dd2b5261762549b37390179f8
   -- https://github.com/lspitzner/czipwith/pull/2
 
-
 source-repository-package
   type: git
   location: https://github.com/jneira/hie-bios/
@@ -67,12 +66,6 @@ source-repository-package
   subdir: dependent-sum-template
 -- https://github.com/obsidiansystems/dependent-sum/pull/57
 
-source-repository-package
-  type: git
-  location: https://github.com/anka-213/HieDb
-  tag: a3f7521f6c5af1b977040cce09c8f7354f8984eb
--- https://github.com/wz1000/HieDb/pull/31
-
 source-repository-package
   type: git
   location: https://github.com/anka-213/lsp
@@ -84,7 +77,7 @@ source-repository-package
 
 write-ghc-environment-files: never
 
-index-state: 2021-06-21T19:57:32Z
+index-state: 2021-06-30T16:00:00Z
 
 constraints:
     -- These plugins doesn't work on GHC9 yet

cabal.project

@@ -36,7 +36,7 @@ source-repository-package
 
 write-ghc-environment-files: never
 
-index-state: 2021-06-21T19:57:32Z
+index-state: 2021-06-30T16:00:00Z
 
 constraints:
     -- Diagrams doesn't support optparse-applicative >= 0.16 yet

ghcide/ghcide.cabal

@@ -2,7 +2,7 @@ cabal-version:      2.4
 build-type:         Simple
 category:           Development
 name:               ghcide
-version:            1.4.0.2
+version:            1.4.0.3
 license:            Apache-2.0
 license-file:       LICENSE
 author:             Digital Asset and Ghcide contributors
@@ -59,7 +59,7 @@ library
         hie-compat ^>= 0.2.0.0,
         hls-plugin-api ^>= 1.1.0.0,
         lens,
-        hiedb == 0.3.0.*,
+        hiedb == 0.4.0.*,
         lsp-types >= 1.2 && < 1.4,
         lsp == 1.2.*,
         mtl,

ghcide/src/Development/IDE/Core/OfInterest.hs

@@ -19,24 +19,19 @@ module Development.IDE.Core.OfInterest(
 import           Control.Concurrent.Strict
 import           Control.Monad
 import           Control.Monad.IO.Class
-import           Data.HashMap.Strict                          (HashMap)
-import qualified Data.HashMap.Strict                          as HashMap
-import qualified Data.Text                                    as T
+import           Data.HashMap.Strict                    (HashMap)
+import qualified Data.HashMap.Strict                    as HashMap
+import qualified Data.Text                              as T
 import           Development.IDE.Graph
 
-import           Control.Monad.Trans.Class
-import           Control.Monad.Trans.Maybe
-import qualified Data.ByteString                              as BS
-import           Data.List.Extra                              (nubOrd)
-import           Data.Maybe                                   (catMaybes)
+import qualified Data.ByteString                        as BS
+import           Data.Maybe                             (catMaybes)
 import           Development.IDE.Core.ProgressReporting
 import           Development.IDE.Core.RuleTypes
 import           Development.IDE.Core.Shake
-import           Development.IDE.Import.DependencyInformation
 import           Development.IDE.Types.Exports
 import           Development.IDE.Types.Location
 import           Development.IDE.Types.Logger
-import           Development.IDE.Types.Options
 
 newtype OfInterestVar = OfInterestVar (Var (HashMap NormalizedFilePath FileOfInterestStatus))
 instance IsIdeGlobal OfInterestVar
@@ -98,25 +93,13 @@ deleteFileOfInterest state f = do
 kick :: Action ()
 kick = do
     files <- HashMap.keys <$> getFilesOfInterestUntracked
-    ShakeExtras{progress} <- getShakeExtras
+    ShakeExtras{exportsMap, progress} <- getShakeExtras
     liftIO $ progressUpdate progress KickStarted
 
-    -- Update the exports map for FOIs
+    -- Update the exports map
     results <- uses GenerateCore files <* uses GetHieAst files
-
-    -- Update the exports map for non FOIs
-    -- We can skip this if checkProject is True, assuming they never change under our feet.
-    IdeOptions{ optCheckProject = doCheckProject } <- getIdeOptions
-    checkProject <- liftIO doCheckProject
-    ifaces <- if checkProject then return Nothing else runMaybeT $ do
-        deps <- MaybeT $ sequence <$> uses GetDependencies files
-        hiResults <- lift $ uses GetModIface (nubOrd $ foldMap transitiveModuleDeps deps)
-        return $ map hirModIface $ catMaybes hiResults
-
-    ShakeExtras{exportsMap} <- getShakeExtras
     let mguts = catMaybes results
         !exportsMap' = createExportsMapMg mguts
-        !exportsMap'' = maybe mempty createExportsMap ifaces
-    void $ liftIO $ modifyVar' exportsMap $ (exportsMap'' <>) . (exportsMap' <>)
+    void $ liftIO $ modifyVar' exportsMap (exportsMap' <>)
 
     liftIO $ progressUpdate progress KickCompleted

ghcide/src/Development/IDE/Core/Shake.hs

@@ -152,6 +152,8 @@ import           Data.HashSet                           (HashSet)
 import qualified Data.HashSet                           as HSet
 import           Data.IORef.Extra                       (atomicModifyIORef'_,
                                                          atomicModifyIORef_)
+import           Data.Text                              (pack)
+import qualified Development.IDE.Types.Exports          as ExportsMap
 import           HieDb.Types
 import           Ide.Plugin.Config
 import qualified Ide.PluginUtils                        as HLS
@@ -507,6 +509,12 @@ shakeOpen lspEnv defaultConfig logger debouncer
         indexProgressToken <- newVar Nothing
         let hiedbWriter = HieDbWriter{..}
         exportsMap <- newVar mempty
+        -- lazily initialize the exports map with the contents of the hiedb
+        _ <- async $ do
+            logDebug logger "Initializing exports map from hiedb"
+            em <- createExportsMapHieDb hiedb
+            modifyVar' exportsMap (<> em)
+            logDebug logger $ "Done initializing exports map from hiedb (" <> pack(show (ExportsMap.size em)) <> ")"
 
         progress <- do
             let (before, after) = if testing then (0,0.1) else (0.1,0.1)

ghcide/src/Development/IDE/Types/Exports.hs

@@ -7,12 +7,13 @@ module Development.IDE.Types.Exports
     createExportsMap,
     createExportsMapMg,
     createExportsMapTc
-) where
+,createExportsMapHieDb,size) where
 
 import           Avail                      (AvailInfo (..))
 import           Control.DeepSeq            (NFData (..))
+import           Control.Monad
 import           Data.Bifunctor             (Bifunctor (second))
-import           Data.HashMap.Strict        (HashMap)
+import           Data.HashMap.Strict        (HashMap, elems)
 import qualified Data.HashMap.Strict        as Map
 import           Data.HashSet               (HashSet)
 import qualified Data.HashSet               as Set
@@ -23,13 +24,17 @@ import           Development.IDE.GHC.Util
 import           FieldLabel                 (flSelector)
 import           GHC.Generics               (Generic)
 import           GhcPlugins                 (IfaceExport, ModGuts (..))
+import           HieDb
 import           Name
 import           TcRnTypes                  (TcGblEnv (..))
 
 newtype ExportsMap = ExportsMap
     {getExportsMap :: HashMap IdentifierText (HashSet IdentInfo)}
     deriving newtype (Monoid, NFData, Show)
 
+size :: ExportsMap -> Int
+size = sum . map length . elems . getExportsMap
+
 instance Semigroup ExportsMap where
     ExportsMap a <> ExportsMap b = ExportsMap $ Map.unionWith (<>) a b
 
@@ -104,6 +109,22 @@ createExportsMapTc = ExportsMap . Map.fromListWith (<>) . concatMap doOne
       where
         mn = moduleName $ tcg_mod mi
 
+createExportsMapHieDb :: HieDb -> IO ExportsMap
+createExportsMapHieDb hiedb = do
+    mods <- getAllIndexedMods hiedb
+    idents <- forM mods $ \m -> do
+        let mn = modInfoName $ hieModInfo m
+            mText = pack $ moduleNameString mn
+        fmap (wrap . unwrap mText) <$> getExportsForModule hiedb mn
+    return $ ExportsMap $ Map.fromListWith (<>) (concat idents)
+  where
+    wrap identInfo = (name identInfo, Set.fromList [identInfo])
+    -- unwrap :: ExportRow -> IdentInfo
+    unwrap m ExportRow{..} = IdentInfo n n p exportIsDatacon m
+      where
+          n = pack (occNameString exportName)
+          p = pack . occNameString <$> exportParent
+
 unpackAvail :: ModuleName -> IfaceExport -> [(Text, [IdentInfo])]
 unpackAvail !(pack . moduleNameString -> mod) = map f . mkIdentInfos mod
   where

stack-8.10.2.yaml

@@ -59,7 +59,7 @@ extra-deps:
   - temporary-1.2.1.1
   - th-compat-0.1.2@sha256:3d55de1adc542c1a870c9ada90da2fbbe5f4e8bcd3eed545a55c3df9311b32a8,2854
   - bytestring-encoding-0.1.0.0@sha256:460b49779fbf0112e8e2f1753c1ed9131eb18827600c298f4d6bb51c4e8c1c0d,1727
-  - hiedb-0.3.0.1
+  - hiedb-0.4.0.0
   - dependent-map-0.4.0.0@sha256:ca2b131046f4340a1c35d138c5a003fe4a5be96b14efc26291ed35fd08c62221,1657
   - dependent-sum-0.7.1.0@sha256:5599aa89637db434431b1dd3fa7c34bc3d565ee44f0519bfbc877be1927c2531,2068
   - dependent-sum-template-0.1.0.3@sha256:0bbbacdfbd3abf2a15aaf0cf2c27e5bdd159b519441fec39e1e6f2f54424adde,1682

stack-8.10.3.yaml

@@ -56,7 +56,7 @@ extra-deps:
   - temporary-1.2.1.1
   - th-compat-0.1.2@sha256:3d55de1adc542c1a870c9ada90da2fbbe5f4e8bcd3eed545a55c3df9311b32a8,2854
   - bytestring-encoding-0.1.0.0@sha256:460b49779fbf0112e8e2f1753c1ed9131eb18827600c298f4d6bb51c4e8c1c0d,1727
-  - hiedb-0.3.0.1
+  - hiedb-0.4.0.0
   - lsp-1.2.0.0
   - lsp-types-1.2.0.0
   - lsp-test-0.14.0.0

stack-8.10.4.yaml

@@ -52,7 +52,7 @@ extra-deps:
   - temporary-1.2.1.1
   - th-compat-0.1.2@sha256:3d55de1adc542c1a870c9ada90da2fbbe5f4e8bcd3eed545a55c3df9311b32a8,2854
   - bytestring-encoding-0.1.0.0@sha256:460b49779fbf0112e8e2f1753c1ed9131eb18827600c298f4d6bb51c4e8c1c0d,1727
-  - hiedb-0.3.0.1
+  - hiedb-0.4.0.0
   - lsp-1.2.0.0
   - lsp-types-1.2.0.0
   - lsp-test-0.14.0.0

stack-8.10.5.yaml

@@ -54,7 +54,7 @@ extra-deps:
   - temporary-1.2.1.1
   - th-compat-0.1.2@sha256:3d55de1adc542c1a870c9ada90da2fbbe5f4e8bcd3eed545a55c3df9311b32a8,2854
   - bytestring-encoding-0.1.0.0@sha256:460b49779fbf0112e8e2f1753c1ed9131eb18827600c298f4d6bb51c4e8c1c0d,1727
-  - hiedb-0.3.0.1
+  - hiedb-0.4.0.0
   - lsp-1.2.0.0
   - lsp-types-1.2.0.0
   - lsp-test-0.14.0.0

stack-8.6.4.yaml

@@ -92,7 +92,7 @@ extra-deps:
   - th-env-0.1.0.2@sha256:d8f1f37f42a8f1a22404d7d0579528af18f5dac7232cca6bdbd5117c115a0ad5,1370
   - th-compat-0.1.2@sha256:3d55de1adc542c1a870c9ada90da2fbbe5f4e8bcd3eed545a55c3df9311b32a8,2854
   - bytestring-encoding-0.1.0.0@sha256:460b49779fbf0112e8e2f1753c1ed9131eb18827600c298f4d6bb51c4e8c1c0d,1727
-  - hiedb-0.3.0.1
+  - hiedb-0.4.0.0
   - extra-1.7.9@sha256:f1dec740f0f2025790c540732bfd52c556ec55bde4f5dfd7cf18e22bd44ff3d0,2683
   - lsp-1.2.0.0
   - lsp-types-1.2.0.0

stack-8.6.5.yaml

@@ -93,7 +93,7 @@ extra-deps:
   - th-env-0.1.0.2@sha256:d8f1f37f42a8f1a22404d7d0579528af18f5dac7232cca6bdbd5117c115a0ad5,1370
   - th-compat-0.1.2@sha256:3d55de1adc542c1a870c9ada90da2fbbe5f4e8bcd3eed545a55c3df9311b32a8,2854
   - bytestring-encoding-0.1.0.0@sha256:460b49779fbf0112e8e2f1753c1ed9131eb18827600c298f4d6bb51c4e8c1c0d,1727
-  - hiedb-0.3.0.1
+  - hiedb-0.4.0.0
   - extra-1.7.9@sha256:f1dec740f0f2025790c540732bfd52c556ec55bde4f5dfd7cf18e22bd44ff3d0,2683
   - lsp-1.2.0.0
   - lsp-types-1.2.0.0

stack-8.8.3.yaml

@@ -70,7 +70,7 @@ extra-deps:
   - uniplate-1.6.13
   - th-compat-0.1.2@sha256:3d55de1adc542c1a870c9ada90da2fbbe5f4e8bcd3eed545a55c3df9311b32a8,2854
   - bytestring-encoding-0.1.0.0@sha256:460b49779fbf0112e8e2f1753c1ed9131eb18827600c298f4d6bb51c4e8c1c0d,1727
-  - hiedb-0.3.0.1
+  - hiedb-0.4.0.0
   - sqlite-simple-0.4.18.0@sha256:3ceea56375c0a3590c814e411a4eb86943f8d31b93b110ca159c90689b6b39e5,3002
   - direct-sqlite-2.3.26@sha256:04e835402f1508abca383182023e4e2b9b86297b8533afbd4e57d1a5652e0c23,3718
   - extra-1.7.9@sha256:f1dec740f0f2025790c540732bfd52c556ec55bde4f5dfd7cf18e22bd44ff3d0,2683

stack-8.8.4.yaml

@@ -68,7 +68,7 @@ extra-deps:
   - temporary-1.2.1.1
   - th-compat-0.1.2@sha256:3d55de1adc542c1a870c9ada90da2fbbe5f4e8bcd3eed545a55c3df9311b32a8,2854
   - bytestring-encoding-0.1.0.0@sha256:460b49779fbf0112e8e2f1753c1ed9131eb18827600c298f4d6bb51c4e8c1c0d,1727
-  - hiedb-0.3.0.1
+  - hiedb-0.4.0.0
   - sqlite-simple-0.4.18.0@sha256:3ceea56375c0a3590c814e411a4eb86943f8d31b93b110ca159c90689b6b39e5,3002
   - direct-sqlite-2.3.26@sha256:04e835402f1508abca383182023e4e2b9b86297b8533afbd4e57d1a5652e0c23,3718
   - lsp-1.2.0.0

stack-9.0.1.yaml

@@ -43,6 +43,7 @@ extra-deps:
 - ghc-lib-parser-ex-9.0.0.4@sha256:8282b11c3797fc8ba225b245e736cc9a0745d9c48d0f9fea7f9bffb5c9997709,3642
 - haddock-library-1.10.0@sha256:2a6c239da9225951a5d837e1ce373faeeae60d1345c78dd0a0b0f29df30c4fe9,4098
 - heapsize-0.3.0.1@sha256:0b69aa97a46d819b700ac7b145f3b5493c3565cf2c5b8298682238d405d0326e,1417
+- hiedb-0.4.0.0
 - hlint-3.3@sha256:4218ad6e03050f5d68aeba0e025f5f05e366c8fd49657f2a19df04ee31b2bb23,4154
 - implicit-hie-0.1.2.5@sha256:517a98ef72f92f0a1617495222774fed3a751a64b0c06fbfc7b858d7aa5de279,2998
 - implicit-hie-cradle-0.3.0.2@sha256:7ad0d10c475ad2b45a068aa0c1b150078ec579746df3b1754d91820354c90696,2594
@@ -81,10 +82,6 @@ extra-deps:
   - dependent-sum-template
 # https://github.com/obsidiansystems/dependent-sum/pull/57
 
-- github: anka-213/HieDb
-  commit: a3f7521f6c5af1b977040cce09c8f7354f8984eb
-# https://github.com/wz1000/HieDb/pull/31
-
 - github: anka-213/lsp
   commit: e96383ab19534128f12acc70a69fbc15d4f298cc
   subdirs:

stack.yaml

@@ -49,7 +49,7 @@ extra-deps:
   - temporary-1.2.1.1
   - th-compat-0.1.2@sha256:3d55de1adc542c1a870c9ada90da2fbbe5f4e8bcd3eed545a55c3df9311b32a8,2854
   - bytestring-encoding-0.1.0.0@sha256:460b49779fbf0112e8e2f1753c1ed9131eb18827600c298f4d6bb51c4e8c1c0d,1727
-  - hiedb-0.3.0.1
+  - hiedb-0.4.0.0
   - lsp-1.2.0.0
   - lsp-types-1.2.0.0
   - lsp-test-0.14.0.0