Use tasty-golden for golden tests and ditch vendored hspec-expectations package.

.travis.yml

@@ -28,7 +28,7 @@ before_install:
 - cabal --version
 
 install:
-- cabal new-update hackage.haskell.org
+- cabal new-update -v hackage.haskell.org
 - cabal new-configure --enable-tests --write-ghc-environment-files=always
 - cabal new-build --only-dependencies -j
 

cabal.project

@@ -14,8 +14,3 @@ source-repository-package
   type: git
   location: https://github.com/joshvera/proto3-wire.git
   tag: 84664e22f01beb67870368f1f88ada5d0ad01f56
-
-source-repository-package
-  type: git
-  location: https://github.com/rewinfrey/hspec-expectations-pretty-diff
-  tag: 94af5871c24ba319f7f72fefa53c1a4d074c9a29

semantic.cabal

@@ -274,7 +274,7 @@ library
   autogen-modules:     Paths_semantic
   other-modules:       Paths_semantic
   build-depends:       base >= 4.12 && < 5
-                     , ansi-terminal ^>= 0.8.2
+                     , ansi-terminal >= 0.8.2 && <1
                      , array ^>= 0.5.3.0
                      , attoparsec ^>= 0.13.2.2
                      , cmark-gfm == 0.1.8
@@ -340,6 +340,7 @@ test-suite test
   import:              haskell, dependencies, executable-flags
   type:                exitcode-stdio-1.0
   hs-source-dirs:      test
+  ghc-options:         -Wunused-imports
   main-is:             Spec.hs
   other-modules:       Analysis.Go.Spec
                      , Analysis.PHP.Spec
@@ -381,13 +382,16 @@ test-suite test
                      , Test.Hspec.LeanCheck
   build-depends:       semantic
                      , tree-sitter-json
-                     , Glob
+                     , Glob ^>= 0.10.0
                      , hspec >= 2.6 && <3
                      , hspec-core >= 2.6 && <3
-                     , hspec-expectations-pretty-diff ^>= 0.7.2.5
+                     , hspec-expectations ^>= 0.8.2
+                     , tasty ^>= 1.2.3
+                     , tasty-golden ^>= 2.3.2
+                     , tasty-hspec ^>= 1.1.5.1
                      , HUnit ^>= 1.6.0.0
                      , leancheck >= 0.8 && <1
-                     , temporary
+                     , temporary ^>= 1.3
   if flag(release)
     ghc-options:       -dynamic
 
@@ -398,9 +402,9 @@ test-suite parse-examples
   main-is:             Examples.hs
   build-depends:       semantic
                      , Glob
-                     , hspec >= 2.4.1
+                     , hspec
                      , hspec-core
-                     , hspec-expectations-pretty-diff
+                     , hspec-expectations
 
 benchmark evaluation
   import:              haskell, executable-flags
@@ -409,7 +413,7 @@ benchmark evaluation
   main-is:             Main.hs
   ghc-options:         -static
   build-depends:       base
-                     , criterion
+                     , criterion ^>= 1.5
                      , semantic
 
 source-repository head

test/Analysis/Go/Spec.hs

@@ -1,10 +1,8 @@
 {-# OPTIONS_GHC -O0 #-}
 module Analysis.Go.Spec (spec) where
 
-import Data.Abstract.Evaluatable (EvalError(..))
 import qualified Data.Abstract.ModuleTable as ModuleTable
 import qualified Data.Language as Language
-import qualified Language.Go.Assignment as Go
 import SpecHelpers
 
 

test/Analysis/PHP/Spec.hs

@@ -1,12 +1,9 @@
 {-# OPTIONS_GHC -O0 #-}
 module Analysis.PHP.Spec (spec) where
 
-import           Control.Abstract
-import           Data.Abstract.Evaluatable (EvalError (..))
 import qualified Data.Abstract.ModuleTable as ModuleTable
 import qualified Data.Abstract.Value.Concrete as Value
 import qualified Data.Language as Language
-import qualified Language.PHP.Assignment as PHP
 import           SpecHelpers
 
 

test/Analysis/Python/Spec.hs

@@ -1,9 +1,8 @@
+{-# OPTIONS_GHC -O0 #-}
 module Analysis.Python.Spec (spec) where
 
-import Data.Abstract.Evaluatable (EvalError(..))
 import qualified Data.Abstract.ModuleTable as ModuleTable
 import Data.Abstract.Value.Concrete
-import qualified Language.Python.Assignment as Python
 import qualified Data.Language as Language
 
 import SpecHelpers

test/Analysis/Ruby/Spec.hs

@@ -2,14 +2,13 @@
 {-# LANGUAGE TupleSections #-}
 module Analysis.Ruby.Spec (spec) where
 
-import           Control.Abstract (Declaration (..), ScopeError (..), runDeref)
+import           Control.Abstract (Declaration (..), ScopeError (..))
 import           Control.Effect.Resumable (SomeError (..))
 import           Data.Abstract.Evaluatable
 import qualified Data.Abstract.ModuleTable as ModuleTable
 import           Data.Abstract.Number as Number
 import           Data.Abstract.Value.Concrete as Value
 import qualified Data.Language as Language
-import           Data.List.NonEmpty (NonEmpty (..))
 import           Data.Sum
 
 import SpecHelpers

test/Analysis/TypeScript/Spec.hs

@@ -5,20 +5,15 @@ module Analysis.TypeScript.Spec (spec) where
 import           Data.Syntax.Statement (StatementBlock(..))
 import qualified Data.Abstract.ScopeGraph as ScopeGraph (AccessControl(..))
 import           Control.Abstract.ScopeGraph hiding (AccessControl(..))
-import           Control.Abstract.Value as Value hiding (String, Unit)
-import           Control.Arrow ((&&&))
 import           Data.Abstract.Evaluatable
 import qualified Data.Abstract.Heap as Heap
 import           Data.Abstract.Module (ModuleInfo (..))
 import qualified Data.Abstract.ModuleTable as ModuleTable
 import           Data.Abstract.Number as Number
 import           Data.Abstract.Package (PackageInfo (..))
-import qualified Data.Abstract.ScopeGraph as ScopeGraph
 import           Data.Abstract.Value.Concrete as Concrete
 import qualified Data.Language as Language
-import qualified Data.List.NonEmpty as NonEmpty
 import           Data.Location
-import qualified Data.Map.Internal as Map
 import           Data.Quieterm
 import           Data.Scientific (scientific)
 import           Data.Sum

test/Control/Abstract/Evaluator/Spec.hs

@@ -8,19 +8,14 @@ import qualified Control.Abstract.Heap as Heap
 import           Data.Abstract.Address.Precise as Precise
 import           Data.Abstract.BaseError
 import           Data.Abstract.Evaluatable
-import           Data.Abstract.FreeVariables
 import           Data.Abstract.Module
 import qualified Data.Abstract.Number as Number
 import           Data.Abstract.Package
 import qualified Data.Abstract.ScopeGraph as ScopeGraph
 import           Data.Abstract.Value.Concrete as Value
-import           Data.Algebra
-import           Data.Bifunctor (first)
-import           Data.Functor.Const
 import qualified Data.Language as Language
 import qualified Data.Map.Strict as Map
 import           Data.Sum
-import           Data.Text (pack)
 import           SpecHelpers hiding (reassociate)
 import           System.IO.Unsafe (unsafePerformIO)
 

test/Data/Abstract/Name/Spec.hs

@@ -2,8 +2,6 @@ module Data.Abstract.Name.Spec where
 
 import SpecHelpers
 
-import Data.Abstract.Name
-
 spec :: Spec
 spec = describe "Data.Abstract.Name" $
   it "should format anonymous names correctly" $ do

test/Data/Term/Spec.hs

@@ -4,7 +4,7 @@ module Data.Term.Spec (spec) where
 import Data.Functor.Listable
 import Data.Term
 import Test.Hspec (Spec, describe, parallel)
-import Test.Hspec.Expectations.Pretty
+import Test.Hspec.Expectations
 import Test.Hspec.LeanCheck
 
 spec :: Spec

test/Diffing/Interpreter/Spec.hs

@@ -15,7 +15,7 @@ import Data.These
 import Diffing.Interpreter
 import qualified Data.Syntax as Syntax
 import Test.Hspec (Spec, describe, it, parallel)
-import Test.Hspec.Expectations.Pretty
+import Test.Hspec.Expectations
 import Test.Hspec.LeanCheck
 import Test.LeanCheck.Core
 import SpecHelpers ()

test/Graphing/Calls/Spec.hs

@@ -8,14 +8,10 @@ import SpecHelpers hiding (readFile)
 import Algebra.Graph
 import Data.List (uncons)
 
-import           Data.Abstract.Module
 import           "semantic" Data.Graph (Graph (..), topologicalSort)
 import           Data.Graph.ControlFlowVertex
-import           Data.Span
 import qualified Data.Language as Language
-import           Semantic.Config (defaultOptions)
 import           Semantic.Graph
-import           Semantic.IO
 
 callGraphPythonProject paths = runTaskOrDie $ do
   let proxy = Proxy @'Language.Python

test/Integration/Spec.hs

@@ -1,36 +1,52 @@
+{-# LANGUAGE ImplicitParams, LambdaCase, NamedFieldPuns #-}
 module Integration.Spec (spec) where
 
 import Control.Exception (throw)
-import Data.Foldable (find, traverse_, for_)
+import Data.Foldable (find)
 import Data.List (union, concat, transpose)
-import qualified Data.ByteString as B
+import qualified Data.ByteString.Lazy as BL
 import System.FilePath.Glob
 import System.FilePath.Posix
+import System.IO.Unsafe
 
 import SpecHelpers
 
+import Test.Tasty
+import Test.Tasty.Golden
+
 languages :: [FilePath]
 languages = ["go", "javascript", "json", "python", "ruby", "typescript", "tsx"]
 
-spec :: TaskSession -> Spec
-spec config = parallel $ do
-  for_ languages $ \language -> do
-    let dir = "test/fixtures" </> language </> "corpus"
-    it (language <> " corpus exists") $ examples dir `shouldNotReturn` []
-    describe (language <> " corpus") $ runTestsIn dir []
+spec :: TaskSession -> TestTree
+spec config = let ?session = config in testGroup "Integration (golden tests)" $ fmap testsForLanguage languages
 
-  where
-    runTestsIn :: FilePath -> [(FilePath, String)] -> SpecWith ()
-    runTestsIn directory pending = do
-      examples <- runIO $ examples directory
-      traverse_ (runTest pending) examples
-    runTest pending ParseExample{..} = it ("parses " <> file) $ maybe (testParse config file parseOutput) pendingWith (lookup parseOutput pending)
-    runTest pending DiffExample{..} = it ("diffs " <> diffOutput) $ maybe (testDiff config (Both fileA fileB) diffOutput) pendingWith (lookup diffOutput pending)
+testsForLanguage :: (?session :: TaskSession) => FilePath -> TestTree
+testsForLanguage language = do
+  let dir = "test/fixtures" </> language </> "corpus"
+  let items = unsafePerformIO (examples dir)
+  localOption (mkTimeout 3000000) $ testGroup language $ fmap testForExample items
+{-# NOINLINE testsForLanguage #-}
 
 data Example = DiffExample { fileA :: FilePath, fileB :: FilePath, diffOutput :: FilePath }
              | ParseExample { file :: FilePath, parseOutput :: FilePath }
              deriving (Eq, Show)
 
+testForExample :: (?session :: TaskSession) => Example -> TestTree
+testForExample = \case
+  DiffExample{fileA, fileB, diffOutput} ->
+    goldenVsStringDiff
+      ("diffs " <> diffOutput)
+      (\ref new -> ["git", "diff", ref, new])
+      diffOutput
+      (BL.fromStrict <$> diffFilePaths ?session (Both fileA fileB))
+  ParseExample{file, parseOutput} ->
+    goldenVsStringDiff
+      ("parses " <> parseOutput)
+      (\ref new -> ["git", "diff", ref, new])
+      parseOutput
+      (parseFilePath ?session file >>= either throw (pure . BL.fromStrict))
+
+
 -- | Return all the examples from the given directory. Examples are expected to
 -- | have the form:
 -- |
@@ -81,18 +97,3 @@ examples directory = do
 -- | Given a test name like "foo.A.js", return "foo".
 normalizeName :: FilePath -> FilePath
 normalizeName path = dropExtension $ dropExtension path
-
-testParse :: TaskSession -> FilePath -> FilePath -> Expectation
-testParse session path expectedOutput = do
-  actual <- fmap verbatim <$> parseFilePath session path
-  case actual of
-    Left err -> throw err
-    Right actual -> do
-      expected <- verbatim <$> B.readFile expectedOutput
-      actual `shouldBe` expected
-
-testDiff :: TaskSession -> Both FilePath -> FilePath -> Expectation
-testDiff config paths expectedOutput = do
-  actual <- verbatim <$> diffFilePaths config paths
-  expected <- verbatim <$> B.readFile expectedOutput
-  actual `shouldBe` expected

test/Parsing/Spec.hs

@@ -1,6 +1,5 @@
 module Parsing.Spec (spec) where
 
-import Control.Effect
 import Data.AST
 import Data.Blob
 import Data.ByteString.Char8 (pack)
@@ -9,9 +8,7 @@ import Data.Language
 import Data.Maybe
 import Data.Source
 import Parsing.TreeSitter
-import Semantic.Config
 import SpecHelpers
-import System.Timeout
 import TreeSitter.JSON (tree_sitter_json, Grammar)
 
 spec :: Spec

test/Rendering/TOC/Spec.hs

@@ -10,20 +10,17 @@ import Data.Diff
 import Data.Functor.Classes
 import Data.Hashable.Lifted
 import Data.Patch
-import Data.Range
 import Data.Location
 import Data.Span
 import Data.Sum
 import Data.Term
 import Data.Text (Text)
-import Data.Text.Encoding (encodeUtf8)
 import Diffing.Algorithm hiding (Diff)
 import Diffing.Interpreter
 import Prelude
 import qualified Data.Syntax as Syntax
 import qualified Data.Syntax.Declaration as Declaration
 import Rendering.TOC
-import Semantic.Config
 import Semantic.Api (diffSummaryBuilder)
 import Serializing.Format as Format
 

test/Reprinting/Spec.hs

@@ -5,23 +5,17 @@ module Reprinting.Spec where
 import SpecHelpers hiding (inject, project)
 
 import           Data.Foldable
-import           Data.Functor.Foldable (cata, embed)
 import qualified Data.Machine as Machine
 
 import           Control.Rewriting hiding (context)
-import           Data.Algebra
-import           Data.Blob
 import qualified Data.Language as Language
 import           Data.Reprinting.Scope
 import           Data.Reprinting.Token
 import           Data.Sum
 import qualified Data.Syntax.Literal as Literal
 import           Language.JSON.PrettyPrint
-import           Language.Python.PrettyPrint
-import           Language.Ruby.PrettyPrint
 import           Reprinting.Pipeline
 import           Reprinting.Tokenize
-import           Semantic.IO
 
 increaseNumbers :: (Literal.Float :< fs, Apply Functor fs) => Rule (Term (Sum fs) History)
 increaseNumbers = do

test/Rewriting/Go/Spec.hs

@@ -3,10 +3,8 @@
 module Rewriting.Go.Spec (spec) where
 
 import           Control.Rewriting
-import           Data.Abstract.Module
 import           Data.List
 import           Data.Sum
-import qualified Data.Syntax.Declaration as Decl
 import qualified Data.Syntax.Literal as Lit
 import qualified Data.Syntax.Statement as Stmt
 import           Data.Text (Text)