Simple benchmark

.github/scripts/configure_project.sh

@@ -14,6 +14,11 @@ fi
 
 echo "package jbeam-edit" >>cabal.project.release.local
 echo "  tests: True" >>cabal.project.release.local
+
+if echo "$MATRIX_FLAGS" | grep -qa ' +transformation '; then
+  echo "  benchmarks: True" >>cabal.project.release.local
+fi
+
 if [[ $IS_EXPERIMENTAL -eq 1 ]]; then
   echo "  flags: $MATRIX_FLAGS" >>cabal.project.release.local
 fi

.github/scripts/prepare_installer.sh

@@ -62,6 +62,8 @@ git show HEAD:"./examples/jbeam/suspension.jbeam" >"$TMP_DIR/suspension.${LABEL}
 echo "fender.jbeam: $TMP_DIR/fender.${LABEL}.jbeam"
 echo "suspension.jbeam: $TMP_DIR/suspension.${LABEL}.jbeam"
 
+cp ./examples/jbeam-edit.yaml ./.jbeam-edit.yaml
+
 ./dist/release/jbeam-edit -i "$TMP_DIR/fender.${LABEL}.jbeam"
 ./dist/release/jbeam-edit -i "$TMP_DIR/suspension.${LABEL}.jbeam"
 
@@ -70,8 +72,8 @@ custom_diff() {
 }
 
 if [[ -n $LABEL ]] && [[ "$LABEL" == "experimental" ]]; then
-  custom_diff "$TMP_DIR/fender.experimental.jbeam" ./examples/transformed_jbeam/fender-cfg-default.jbeam
-  custom_diff "$TMP_DIR/suspension.experimental.jbeam" ./examples/transformed_jbeam/suspension-cfg-default.jbeam
+  custom_diff "$TMP_DIR/fender.experimental.jbeam" ./examples/transformed_jbeam/fender-cfg-example.jbeam
+  custom_diff "$TMP_DIR/suspension.experimental.jbeam" ./examples/transformed_jbeam/suspension-cfg-example.jbeam
 else
   custom_diff "$TMP_DIR/fender.stable.jbeam" ./examples/formatted_jbeam/fender-minimal-jbfl.jbeam
   custom_diff "$TMP_DIR/suspension.stable.jbeam" ./examples/formatted_jbeam/suspension-minimal-jbfl.jbeam

.github/workflows/build-and-release.yaml

@@ -88,6 +88,11 @@ jobs:
           cabal install exe:jbeam-lsp-test-server --project-file cabal.project.release || true
       - name: Run tests (GHC ${{ matrix.ghc }})
         run: cabal test --project-file cabal.project.release
+      - name: Run benchmarks (GHC ${{ matrix.ghc }})
+        if: contains(matrix.flags, ' +transformation ')
+        run: |
+          echo "Running benchmarks for jbeam-edit"
+          cabal bench --project-file cabal.project.release --benchmark-options="--verbosity=1"
       - name: Enforce CRLF newlines on windows
         run: bash ./.github/scripts/replace_newlines.sh
         shell: bash

.github/workflows/future-proofing.yaml

@@ -52,8 +52,9 @@ jobs:
             ${{ runner.os }}-cabal-${{ steps.get-ghc.outputs.ghc-version }}-
       - name: Build project (GHC latest)
         run: cabal build --project-file cabal.project.ci all
-      - name: Build LSP test server
-        continue-on-error: true
-        run: cabal install exe:jbeam-lsp-test-server --project-file cabal.project.ci
       - name: Run tests (GHC latest)
         run: cabal test --project-file cabal.project.ci
+      - name: Run benchmarks (GHC ${{ steps.get-ghc.outputs.ghc-version }})
+        run: |-
+          echo "Running benchmarks for jbeam-edit"
+          cabal bench --project-file cabal.project.ci --benchmark-options="--verbosity=1"

cabal.project.ci

@@ -1,5 +1,6 @@
 import: cabal.project
 test-show-details: direct
+optimization: True
 
 package jbeam-edit
   tests: True

hie.yaml

@@ -10,6 +10,8 @@ cradle:
         component: jbeam-edit:lib:jbeam-language-server
       - path: ./exe/jbeam-edit/Main.hs
         component: jbeam-edit:exe:jbeam-edit
+      - path: ./tools/bench/Main.hs
+        component: jbeam-edit:bench:jbeam-edit-bench
       - path: ./exe/jbeam-edit/CommandLineOptions.hs
         component: jbeam-edit:exe:jbeam-edit
       - path: ./tools/dump_ast/Main.hs

jbeam-edit.cabal

@@ -443,3 +443,36 @@ test-suite jbeam-language-server-test
 
     else
         buildable: False
+
+benchmark jbeam-edit-bench
+    type:               exitcode-stdio-1.0
+    main-is:            Main.hs
+    hs-source-dirs:     tools/bench
+    other-modules:      Paths_jbeam_edit
+    autogen-modules:    Paths_jbeam_edit
+    default-language:   GHC2021
+    default-extensions: OverloadedStrings ImportQualifiedPost
+    ghc-options:
+        -Wall -Wcompat -Widentities -Wincomplete-record-updates
+        -Wincomplete-uni-patterns -Wmissing-export-lists
+        -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints
+
+    build-depends:
+        base,
+        bytestring,
+        containers,
+        directory,
+        filepath,
+        jbeam-edit,
+        mtl,
+        scientific,
+        text >=2.0,
+        vector
+
+    if flag(transformation)
+        build-depends:
+            criterion,
+            jbeam-edit-transformation
+
+    else
+        buildable: False

package.yaml

@@ -152,6 +152,18 @@ executables:
     ghc-options: [-threaded, -rtsopts, -with-rtsopts=-N]
     dependencies: [jbeam-edit]
 
+benchmarks:
+  jbeam-edit-bench:
+    main: Main.hs
+    source-dirs: tools/bench
+    dependencies: [base, jbeam-edit]
+    when:
+      - condition: flag(transformation)
+        then:
+          dependencies: [criterion, jbeam-edit-transformation]
+        else:
+          buildable: false
+
 _jbeam-test-common: &jbeam-test-common
   main: Spec.hs
   ghc-options: [-threaded, -rtsopts, -with-rtsopts=-N]

tools/bench/.dir-locals.el

@@ -0,0 +1,4 @@
+((haskell-mode
+  . ((haskell-process-type . cabal-repl)
+     (eval . (setq-local haskell-process-args-cabal-repl
+                         (append '("jbeam-edit:bench:jbeam-edit-bench" "--project-file" "cabal.project.dev") haskell-process-args-cabal-repl))))))

tools/bench/Main.hs

@@ -0,0 +1,55 @@
+module Main (main) where
+
+import Criterion.Main
+import Data.List (isSuffixOf)
+import Data.Map qualified as M
+import Data.Text qualified as T
+import JbeamEdit.Formatting
+import JbeamEdit.Transformation
+import JbeamEdit.Transformation.Config
+import System.Directory
+import System.FilePath
+
+loadAst :: Read a => FilePath -> IO a
+loadAst path = read <$> readFile path
+
+main :: IO ()
+main = do
+  cwd <- getCurrentDirectory
+  let examplesDir = cwd </> "examples"
+      astDir = examplesDir </> "ast"
+      jbeamAstDir = astDir </> "jbeam"
+      jbflAstDir = astDir </> "jbfl"
+
+  let cfg = newTransformationConfig
+
+  jbeamFiles <- filter (isSuffixOf ".hs") <$> listDirectory jbeamAstDir
+  ruleFiles <- filter (isSuffixOf ".hs") <$> listDirectory jbflAstDir
+
+  jbeamAsts <- mapM (\f -> loadAst (jbeamAstDir </> f)) jbeamFiles
+  ruleAsts <- mapM (\f -> loadAst (jbflAstDir </> f)) ruleFiles
+
+  let combos =
+        [ (jFile, rFile, jAst, rAst)
+        | (jFile, jAst) <- zip jbeamFiles jbeamAsts
+        , (rFile, rAst) <- zip ruleFiles ruleAsts
+        ]
+
+  let benchFormat (jName, rName, jAst, rAst) =
+        bench (jName ++ " + " ++ rName ++ " [format]") $
+          nf (T.length . formatNode rAst) jAst
+
+  let benchTransformFormat (jName, rName, jAst, rAst) =
+        bench (jName ++ " + " ++ rName ++ " [transform+format]") $
+          nf
+            ( \ast ->
+                case transform M.empty cfg ast of
+                  Right (_, _, outAst) -> T.length (formatNode rAst outAst)
+                  Left err -> error (T.unpack err)
+            )
+            jAst
+
+  defaultMain
+    [ bgroup "format-only" (map benchFormat combos)
+    , bgroup "transform+format" (map benchTransformFormat combos)
+    ]