Make BenchHist non buildable by default and save logs (haskell/ghcide…

…#666)

* [bench-hist] save messages to log file

And fix the commitid rule to always rerun the git query

* Do not build benchHist by default

Hopefully avoiding the additional dependencies for charts

* Simplify with FileStdout

* Add a flag for bench-hist

Could benchHist be a benchmark instead of an executable, removing the need
for this flag?

Almost. `stack bench` fails because `benchHist` cannot find `ghcide-bench`
in the path. It seems like a bad idea to have a benchmark that fails out of
the box

* Turn benchHist into a benchmark and ghcide-bench into an exe

This works out nicely because:

1. benchHist already runs ghcide-bench,
2. benchHist has additional deps, but ghcide-bench does not.
   (benchmark deps don't get built by default)
3. This is the only way I've found to get ghcide-bench in the PATH for benchHist

* Remove redundant dep on applicative-combinators

* Bump versions in stack-ghc-lib.yaml

* update lower bounds for extra in benchHist executable

* Update README guideline on benchmarks

* [benchHist] Fix the commitid rule to always rerun the git query

* fix caps

ghcide/README.md

@@ -312,12 +312,14 @@ See the [tasty-rerun](https://hackage.haskell.org/package/tasty-rerun-1.1.17/doc
 
 If you are touching performance sensitive code, take the time to run a differential
 benchmark between HEAD and upstream using the benchHist script. The configuration in
-`bench/hist.yaml` is setup to do this by default with the command:
+`bench/hist.yaml` is setup to do this by default assuming upstream is
+`origin/master`. Run the benchmarks with `stack`:
 
-    stack build ghcide:benchHist && stack exec benchHist
+    export STACK_YAML=...
+    stack bench
 
 It should take around 15 minutes and the results will be stored in the `bench-hist` folder.
-To interpret the results, see the comments in the `bench/Hist/Main.hs` module.
+To interpret the results, see the comments in the `bench/hist/Main.hs` module.
 
 ### Building the extension
 

ghcide/bench/exe/Main.hs

@@ -23,8 +23,8 @@
      - Analyisis of performance over the commit history of the project
 
   How to run:
-     1. `cabal bench`
-     2. `cabal exec cabal run ghcide-bench -- -- ghcide-bench-options`
+     1. `cabal exec cabal run ghcide-bench -- -- ghcide-bench-options`
+     1. `stack build ghcide:ghcide-bench && stack exec ghcide-bench -- -- ghcide-bench-options`
 
   Note that the package database influences the response times of certain actions,
   e.g. code actions, and therefore the two methods above do not necessarily

ghcide/bench/hist.yaml

@@ -36,7 +36,7 @@ versions:
 # - v0.0.5
 # - v0.0.6
 # - v0.1.0
-- v0.2.0
-- upstream: origin/master
+# - v0.2.0
+- upstream: upstream/master
 - HEAD
 

ghcide/bench/hist/Main.hs

@@ -29,11 +29,11 @@
 
    To execute the script:
 
-   > stack build ghcide:exe:benchHist && stack exec benchHist all
+   > stack bench
 
    To build a specific analysis, enumerate the desired file artifacts
 
-   > stack exec benchHist bench-hist/HEAD/results.csv bench-hist/HEAD/edit.diff.svg
+   > stack bench --ba "bench-hist/HEAD/results.csv bench-hist/HEAD/edit.diff.svg"
 
  -}
 {-# LANGUAGE DeriveAnyClass    #-}
@@ -124,6 +124,7 @@ main = shakeArgs shakeOptions {shakeChange = ChangeModtimeAndDigest} $ do
            ]
 
   build -/- "*/commitid" %> \out -> do
+      alwaysRerun
 
       let [_,ver,_] = splitDirectories out
       mbEntry <- find ((== T.pack ver) . humanName) <$> readVersions
@@ -178,37 +179,36 @@ main = shakeArgs shakeOptions {shakeChange = ChangeModtimeAndDigest} $ do
 
   priority 0 $
     [ build -/- "*/*.csv",
-      build -/- "*/*.benchmark-gcStats"
+      build -/- "*/*.benchmark-gcStats",
+      build -/- "*/*.log"
     ]
-      &%> \[outcsv, _outGc] -> do
+      &%> \[outcsv, _outGc, outLog] -> do
         let [_, _, exp] = splitDirectories outcsv
         samples <- readSamples
         liftIO $ createDirectoryIfMissing True $ dropFileName outcsv
         let ghcide = dropFileName outcsv </> "ghcide"
             ghcpath = dropFileName outcsv </> "ghc.path"
         need [ghcide, ghcpath]
         ghcPath <- readFile' ghcpath
-        verb <- getVerbosity
         withResource ghcideBenchResource 1 $ do
-          Stdout res <-
-            command
-              [ EchoStdout True,
+          command_
+              [ EchoStdout False,
+                FileStdout outLog,
                 RemEnv "NIX_GHC_LIBDIR",
                 RemEnv "GHC_PACKAGE_PATH",
                 AddPath [takeDirectory ghcPath, "."] []
               ]
               ghcideBenchPath
               [ "--timeout=3000",
+                "-v",
                 "--samples=" <> show samples,
                 "--csv=" <> outcsv,
                 "--example-package-version=3.0.0.0",
                 "--rts=-I0.5",
                 "--ghcide=" <> ghcide,
                 "--select",
-                unescaped (unescapeExperiment (Escaped $ dropExtension exp)),
-                if verb > Normal then "-v" else "-q"
+                unescaped (unescapeExperiment (Escaped $ dropExtension exp))
               ]
-          writeFile' (replaceExtension outcsv "log") res
           cmd_ Shell $ "mv *.benchmark-gcStats " <> dropFileName outcsv
 
   build -/- "results.csv" %> \out -> do

ghcide/ghcide.cabal

@@ -173,11 +173,14 @@ executable ghcide-test-preprocessor
     build-depends:
         base == 4.*
 
-executable benchHist
+benchmark benchHist
+    type: exitcode-stdio-1.0
     default-language: Haskell2010
-    buildable: True
     ghc-options: -Wall -Wno-name-shadowing -threaded
     main-is: bench/hist/Main.hs
+    build-tool-depends:
+        ghcide:ghcide,
+        ghcide:ghcide-bench
     default-extensions:
         BangPatterns
         DeriveFunctor
@@ -201,7 +204,7 @@ executable benchHist
         diagrams,
         diagrams-svg,
         directory,
-        extra,
+        extra >= 1.7.2,
         filepath,
         shake,
         text,
@@ -306,7 +309,6 @@ test-suite ghcide-tests
         lens,
         lsp-test >= 0.11.0.1 && < 0.12,
         optparse-applicative,
-        parser-combinators,
         process,
         QuickCheck,
         quickcheck-instances,
@@ -342,12 +344,10 @@ test-suite ghcide-tests
         TypeApplications
         ViewPatterns
 
-benchmark ghcide-bench
-    type: exitcode-stdio-1.0
+executable ghcide-bench
     default-language: Haskell2010
     build-tool-depends:
-        ghcide:ghcide,
-        ghcide:ghcide-test-preprocessor
+        ghcide:ghcide
     build-depends:
         aeson,
         base,
@@ -359,7 +359,6 @@ benchmark ghcide-bench
         ghcide,
         lsp-test >= 0.11.0.2 && < 0.12,
         optparse-applicative,
-        parser-combinators,
         process,
         safe-exceptions
     hs-source-dirs: bench/lib bench/exe

ghcide/hie.yaml

@@ -13,8 +13,8 @@ cradle:
             - path: "./test"
               component: "ghcide:test:ghcide-tests"
             - path: "./bench"
-              component: "ghcide:bench:ghcide-bench"
-            - path: "./bench/Hist"
-              component: "ghcide:exe:benchHist"
+              component: "ghcide:exe:ghcide-bench"
+            - path: "./bench/hist"
+              component: "ghcide:bench:benchHist"
             - path: "./test/preprocessor"
               component: "ghcide:exe:ghcide-test-preprocessor"

ghcide/hie.yaml.stack

@@ -13,8 +13,8 @@ cradle:
             - path: "./test"
               component: "ghcide:test:ghcide-tests"
             - path: "./bench"
-              component: "ghcide:bench:ghcide-bench"
+              component: "ghcide:exe:ghcide-bench"
             - path: "./bench/Hist"
-              component: "ghcide:exe:benchHist"
+              component: "ghcide:bench:benchHist"
             - path: "./test/preprocessor"
               component: "ghcide:exe:ghcide-test-preprocessor"

ghcide/stack-ghc-lib.yaml

@@ -4,7 +4,8 @@ packages:
 extra-deps:
 - haskell-lsp-0.22.0.0
 - haskell-lsp-types-0.22.0.0
-- lsp-test-0.10.3.0
+- lsp-test-0.11.0.2
+- extra-1.7.2
 - hie-bios-0.5.0
 - ghc-lib-parser-8.8.1
 - ghc-lib-8.8.1
@@ -14,6 +15,7 @@ extra-deps:
 - regex-tdfa-1.3.1.0
 - haddock-library-1.8.0
 - ghc-check-0.5.0.1
+- parser-combinators-1.2.1
 nix:
   packages: [zlib]
 flags: