From debcb33f2c896732c58dcfc904c96715433861fa Mon Sep 17 00:00:00 2001 From: Hernan Rajchert Date: Tue, 22 Nov 2022 16:39:29 -0300 Subject: [PATCH] Re-add ci tests --- .github/workflows/test.yml | 66 +++++++------------------------------- flake.nix | 37 ++++++++++++++++++--- nix/formatting.nix | 35 ++++++++++++++++++++ nix/scripts.nix | 47 +++++++++++++++------------ nix/tests/default.nix | 50 +++++++++++++++++++++++++++++ nix/tests/generated.nix | 27 ++++++++++++++++ nix/tests/nixpkgs-fmt.nix | 21 ++++++++++++ nix/tests/prettier.nix | 46 ++++++++++++++++++++++++++ 8 files changed, 249 insertions(+), 80 deletions(-) create mode 100644 nix/formatting.nix create mode 100644 nix/tests/default.nix create mode 100644 nix/tests/generated.nix create mode 100644 nix/tests/nixpkgs-fmt.nix create mode 100644 nix/tests/prettier.nix diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2432ca9dae..aefcf70c0e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,92 +14,48 @@ on: - shlevy workflow_call: jobs: - nix-instantiate: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2.3.4 - - uses: cachix/install-nix-action@v16 - with: - nix_path: nixpkgs=channel:nixos-unstable - install_url: https://releases.nixos.org/nix/nix-2.5.1/install - extra_nix_config: | - allowed-uris = ${{ env.ALLOWED_URIS }} - trusted-public-keys = ${{ env.TRUSTED_PUBLIC_KEYS }} - substituters = ${{ env.SUBSTITUTERS }} - experimental-features = nix-command flakes - - uses: cachix/cachix-action@v10 - with: - name: ${{ env.CACHE_NAME }} - authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" - - run: nix-instantiate release.nix --arg checkMaterialization true --arg supportedSystems '[ builtins.currentSystem ]' --restrict-eval -I . nix-code-formatting: strategy: matrix: os: [ubuntu-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v3 - uses: cachix/install-nix-action@v16 with: nix_path: nixpkgs=channel:nixos-unstable - install_url: https://releases.nixos.org/nix/nix-2.5.1/install + install_url: https://releases.nixos.org/nix/nix-2.8.1/install extra_nix_config: | allowed-uris = ${{ env.ALLOWED_URIS }} trusted-public-keys = ${{ env.TRUSTED_PUBLIC_KEYS }} substituters = ${{ env.SUBSTITUTERS }} experimental-features = nix-command flakes - - uses: cachix/cachix-action@v10 + - uses: cachix/cachix-action@v12 with: name: ${{ env.CACHE_NAME }} authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" - - run: nix-build -A tests.nixpkgsFmt -A tests.pursTidy -A tests.prettier -A tests.shellcheck -A tests.stylishHaskell --arg supportedSystems '[ builtins.currentSystem ]' --restrict-eval -I . + - run: nix build .#test-nix-fmt .#test-prettier + # - run: nix-build -A tests.nixpkgsFmt -A tests.pursTidy -A tests.prettier -A tests.shellcheck -A tests.stylishHaskell --arg supportedSystems '[ builtins.currentSystem ]' --restrict-eval -I . nix-code-generation: strategy: matrix: os: [ubuntu-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2.3.4 - - uses: cachix/install-nix-action@v16 - with: - nix_path: nixpkgs=channel:nixos-unstable - install_url: https://releases.nixos.org/nix/nix-2.5.1/install - extra_nix_config: | - allowed-uris = ${{ env.ALLOWED_URIS }} - trusted-public-keys = ${{ env.TRUSTED_PUBLIC_KEYS }} - substituters = ${{ env.SUBSTITUTERS }} - experimental-features = nix-command flakes - - uses: cachix/cachix-action@v10 - with: - name: ${{ env.CACHE_NAME }} - authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" - - run: nix-build -A tests.generated --arg supportedSystems '[ builtins.currentSystem ]' --restrict-eval -I . - check-for-updates: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v3 - uses: cachix/install-nix-action@v16 with: nix_path: nixpkgs=channel:nixos-unstable - install_url: https://releases.nixos.org/nix/nix-2.5.1/install + install_url: https://releases.nixos.org/nix/nix-2.8.1/install extra_nix_config: | allowed-uris = ${{ env.ALLOWED_URIS }} trusted-public-keys = ${{ env.TRUSTED_PUBLIC_KEYS }} substituters = ${{ env.SUBSTITUTERS }} experimental-features = nix-command flakes - - uses: cachix/cachix-action@v10 + - uses: cachix/cachix-action@v12 with: name: ${{ env.CACHE_NAME }} authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" - - run: | - nix flake lock - nix-shell --command "update-client-deps" - # Disabled until we can find a way to run it only for Linux - # nix-shell --command "updateMaterialized" --option trusted-public-keys "${{ env.TRUSTED_PUBLIC_KEYS }}" --option substituters "${{ env.SUBSTITUTERS }}" - git diff --exit-code + - run: nix build .#test-generated + # - run: nix-build -A tests.generated --arg supportedSystems '[ builtins.currentSystem ]' --restrict-eval -I . + diff --git a/flake.nix b/flake.nix index 9d87effed7..86bf746dee 100644 --- a/flake.nix +++ b/flake.nix @@ -23,11 +23,28 @@ p = pkgs.callPackage (easy-purescript-nix) { }; in p // { purs = p.purs-0_15_2; }; + writeShellScriptBinInRepoRoot = name: script: pkgs.writeShellScriptBin name '' + cd `${pkgs.git}/bin/git rev-parse --show-toplevel` + ${script} + ''; scripts = import ./nix/scripts.nix { - inherit pkgs easyPS; + inherit pkgs easyPS writeShellScriptBinInRepoRoot; inherit (pkgs.nodePackages) prettier; + + }; + + formatting = import ./nix/formatting.nix { + inherit writeShellScriptBinInRepoRoot pkgs easyPS; + }; + + tests = import ./nix/tests/default.nix { + inherit pkgs; + inherit (formatting) fix-prettier; + src = ./.; + play-generated = scripts.generated-purescript; }; + overlays = [ haskellNix.overlay (final: prev: { @@ -48,8 +65,13 @@ ) ++ (with scripts; [ - marlowePlaygroundGeneratePurs - startBackend + marlowe-playground-generate-purs + start-backend + ] + ) + ++ + (with formatting; [ + fix-prettier ] ) ++ @@ -68,6 +90,7 @@ }; }) ]; + pkgs = import nixpkgs { inherit system overlays; inherit (haskellNix) config; }; flake = pkgs.playground.flake { }; ghc-with-marlowe = pkgs.playground.ghcWithPackages (ps: [ ps.marlowe ]); @@ -77,7 +100,13 @@ legacyPackages = pkgs; }) { - packages = { inherit ghc-with-marlowe; }; + packages = { + inherit ghc-with-marlowe; + inherit (scripts) marlowe-playground-generate-purs generated-purescript; + test-nix-fmt = tests.nixpkgsFmt; + test-prettier = tests.prettier; + test-generated = tests.generated; + }; } ); diff --git a/nix/formatting.nix b/nix/formatting.nix new file mode 100644 index 0000000000..9f38b4f26a --- /dev/null +++ b/nix/formatting.nix @@ -0,0 +1,35 @@ +{ writeShellScriptBinInRepoRoot +, pkgs +, easyPS +}: +let + inherit (builtins) concatStringsSep; + inherit (pkgs.nodePackages) prettier; + inherit (easyPS) purs-tidy; + extensionsToRegex = extensions: "\\.(${concatStringsSep "|" extensions})"; + writeFormatter = name: cmd: extensions: writeShellScriptBinInRepoRoot "fix-${name}" '' + set -e + echo formatting with ${name} + ${pkgs.git}/bin/git ls-files ':!:bitte/node/config/*'\ + | grep -E '${extensionsToRegex extensions}' \ + | xargs -d $'\\n' ${cmd} + echo done. + ''; +in +{ + fix-prettier = writeFormatter + "prettier" + "${prettier}/bin/prettier -w" + [ "js" "ts" "css" "html" ]; + fix-purs-tidy = writeFormatter + "purs-tidy" + "${purs-tidy}/bin/purs-tidy format-in-place" + [ "purs" ]; + purs-tidy-hook = { + enable = true; + name = "purs-tidy"; + entry = "${purs-tidy}/bin/purs-tidy format-in-place"; + files = "\\.purs$"; + language = "system"; + }; +} diff --git a/nix/scripts.nix b/nix/scripts.nix index cc08120ebd..b3c1e10403 100644 --- a/nix/scripts.nix +++ b/nix/scripts.nix @@ -1,40 +1,45 @@ -{ pkgs, easyPS, prettier }: +{ pkgs, easyPS, prettier, writeShellScriptBinInRepoRoot }: let - writeShellScriptBinInRepoRoot = name: script: pkgs.writeShellScriptBin name '' - cd `${pkgs.git}/bin/git rev-parse --show-toplevel` - ${script} + + # Output containing the purescript bridge code + generated-purescript = pkgs.runCommand "marlowe-playground-purescript" { } '' + mkdir $out + ${pkgs.playground.hsPkgs.marlowe-playground-server.components.exes.marlowe-playground-server}/bin/marlowe-playground-server psgenerator $out + cp ${builtins.path { name = "tidyrc.json"; path = ../.tidyrc.json; } } $out/.tidyrc.json + cp ${builtins.path { name = "tidyoperators"; path = ../.tidyoperators; } } $out/.tidyoperators + cd $out + ${easyPS.purs-tidy}/bin/purs-tidy format-in-place * + ${prettier}/bin/prettier -w . + rm -f $out/.tidyrc.json + rm -f $out/.tidyoperators ''; - marlowePlaygroundGeneratePurs = writeShellScriptBinInRepoRoot "marlowe-playground-generate-purs" '' - #!/bin/bash + # generate-purescript = writeShellScriptBinInRepoRoot "marlowe-playground-generate-purs" '' + # generated=./marlowe-playground-client/generated + # rm -rf $generated + # cp -a $(nix-build -A marlowe-playground.generated-purescript --no-out-link "$@") $generated + # chmod -R +w $generated + # ''; + + + marlowe-playground-generate-purs = writeShellScriptBinInRepoRoot "marlowe-playground-generate-purs" '' generated=./marlowe-playground-client/generated # Clean old build rm -rf $generated # Re-generate the files - mkdir -p $generated - nix run .#marlowe-playground-server:exe:marlowe-playground-server -- psgenerator $generated - - # Format them - cp ${builtins.path { name = "tidyrc.json"; path = ../.tidyrc.json; } } $generated/.tidyrc.json - cp ${builtins.path { name = "tidyoperators"; path = ../.tidyoperators; } } $generated/.tidyoperators - cd $generated - ${easyPS.purs-tidy}/bin/purs-tidy format-in-place * - ${prettier}/bin/prettier -w . - rm -f .tidyrc.json - rm -f .tidyoperators - cd ../.. + cp -a $(nix build .#generated-purescript --no-link --print-out-paths) $generated chmod -R +w $generated ''; - build-ghc-with-marlowe = "$(nix build .#ghc-with-marlowe --print-out-paths)"; + build-ghc-with-marlowe = "$(nix build .#ghc-with-marlowe --no-link --print-out-paths)"; # start-backend: script to start the plutus-playground-server # # Note: We need to add ghc to the path because the server provides /runghc # which needs ghc and dependencies. - startBackend = writeShellScriptBinInRepoRoot "marlowe-playground-server" '' + start-backend = writeShellScriptBinInRepoRoot "marlowe-playground-server" '' echo "marlowe-playground-server: for development use only" GHC_WITH_PKGS=${build-ghc-with-marlowe} export PATH=$GHC_WITH_PKGS/bin:$PATH @@ -43,5 +48,5 @@ let ''; in { - inherit marlowePlaygroundGeneratePurs startBackend; + inherit marlowe-playground-generate-purs start-backend generated-purescript; } diff --git a/nix/tests/default.nix b/nix/tests/default.nix new file mode 100644 index 0000000000..c087d2729f --- /dev/null +++ b/nix/tests/default.nix @@ -0,0 +1,50 @@ +{ pkgs + # , gitignore-nix + # , fixStylishHaskell + # , fix-purs-tidy +, fix-prettier + # , fixPngOptimization +, src + # , marlowe-playground +, play-generated + # , web-ghc + # , docs + # , sources + # , vmCompileTests ? false +}: +{ + # shellcheck = pkgs.callPackage ./shellcheck.nix { src = cleanSrc; }; + + # stylishHaskell = pkgs.callPackage ./stylish-haskell.nix { + # src = cleanSrc; + # inherit fixStylishHaskell; + # }; + + generated = pkgs.callPackage ./generated.nix { + inherit src play-generated; + }; + + # pursTidy = pkgs.callPackage ./purs-tidy.nix { + # src = cleanSrc; + # inherit fix-purs-tidy; + # }; + + prettier = pkgs.callPackage ./prettier.nix { + inherit src fix-prettier; + }; + + nixpkgsFmt = pkgs.callPackage ./nixpkgs-fmt.nix { + inherit src; + inherit (pkgs) nixpkgs-fmt; + }; + + # pngOptimization = pkgs.callPackage ./png-optimization.nix { + # src = cleanSrc; + # inherit fixPngOptimization; + # }; + + # vmTests = pkgs.callPackage ./vm.nix { + # inherit vmCompileTests marlowe-playground + # web-ghc docs sources; + # }; +} diff --git a/nix/tests/generated.nix b/nix/tests/generated.nix new file mode 100644 index 0000000000..937caafa1f --- /dev/null +++ b/nix/tests/generated.nix @@ -0,0 +1,27 @@ +{ runCommand, play-generated, src, diffutils, glibcLocales }: +runCommand "generated-purescript-test" +{ + buildInputs = [ diffutils glibcLocales ]; +} '' + set +e + echo ${toString src} + cp -a ${src} expected + cp -a ${src} actual + chmod -R +w actual + rm -rf actual/marlowe-playground-client/generated + cp -a ${play-generated} actual/marlowe-playground-client/generated + diff --brief --recursive expected actual + EXIT_CODE=$? + if [[ $EXIT_CODE != 0 ]] + then + mkdir -p $out/nix-support + diff -ur expected actual > $out/actual.diff + echo "file none $out/actual.diff" > $out/nix-support/hydra-build-products + echo "*** actual found changes that need addressed first" + echo "*** Please run \`generate-purescript\` and commit changes." + echo "*** or apply the diff generated by hydra if you don't have nix." + exit $EXIT_CODE + else + echo $EXIT_CODE > $out + fi +'' diff --git a/nix/tests/nixpkgs-fmt.nix b/nix/tests/nixpkgs-fmt.nix new file mode 100644 index 0000000000..d5e568691a --- /dev/null +++ b/nix/tests/nixpkgs-fmt.nix @@ -0,0 +1,21 @@ +{ runCommand, nixpkgs-fmt, src }: + +# Runs `nixpkgs-fmt --check` on ${src}. If nixpkgs-fmt +# reports that files need to be re-formatted details are +# written to `$out/nix-support/hydra-build-products` +runCommand "nixpkgs-fmt" +{ + buildInputs = [ nixpkgs-fmt ]; +} '' + set +e + nixpkgs-fmt --check ${src} 2>&1 >nixpkgs-fmt.log + if [ $? -ne 0 ]; then + mkdir -p $out/nix-support + cat nixpkgs-fmt.log > $out/nix-support/hydra-build-products + echo "*** nixpkgs-fmt found files that haven't been formatted" + echo "*** Please run \`nix-shell --run \"nixpkgs-fmt .\"\` and commit the changes" + exit 1 + else + echo 0 > $out + fi +'' diff --git a/nix/tests/prettier.nix b/nix/tests/prettier.nix new file mode 100644 index 0000000000..3cb97cb5a0 --- /dev/null +++ b/nix/tests/prettier.nix @@ -0,0 +1,46 @@ +{ runCommand, fix-prettier, src, lib, diffutils, glibcLocales }: +let + # just JavaScript, HTML, and CSS sources + src' = lib.cleanSourceWith { + inherit src; + filter = with lib; + name: type: + let baseName = baseNameOf (toString name); in + ( + (type == "regular" && hasSuffix ".js" baseName) || + (type == "regular" && hasSuffix ".ts" baseName) || + (type == "regular" && hasSuffix ".html" baseName) || + (type == "regular" && hasSuffix ".css" baseName) || + (type == "directory" && (baseName != "generated" + && baseName != "output" + && baseName != "node_modules" + && baseName != ".spago")) + ); + }; +in +runCommand "prettier-check" +{ + buildInputs = [ fix-prettier diffutils glibcLocales ]; +} '' + set +e + cp -a ${src'} orig + cp -a ${src'} prettier + chmod -R +w prettier + cd prettier + fix-prettier + cd .. + diff --brief --recursive orig prettier > /dev/null + EXIT_CODE=$? + if [[ $EXIT_CODE != 0 ]] + then + mkdir -p $out/nix-support + diff -ur orig prettier > $out/prettier.diff + echo "file none $out/prettier.diff" > $out/nix-support/hydra-build-products + echo "*** prettier found changes that need addressed first" + echo "*** Please run \`nix-shell --run fix-prettier\` and commit changes" + echo "*** or apply the diff generated by hydra if you don't have nix." + exit $EXIT_CODE + else + echo $EXIT_CODE > $out + fi +''