Add postgrest-coverage to show and upload hpc reports to codecov

PostgREST · Dec 30, 2020 · c7f0d42 · c7f0d42
1 parent 2015688
commit c7f0d42
Show file tree

Hide file tree

Showing 17 changed files with 179 additions and 69 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -234,6 +234,13 @@ jobs:
           name: Run memory tests
           command: postgrest-test-memory
           when: always
+      - run:
+          name: Run coverage
+          command: postgrest-coverage
+      - run:
+          name: Upload coverage to codecov
+          command: bash <(curl -s https://codecov.io/bash) -f coverage/codecov.json
+
 
 workflows:
   version: 2

diff --git a/.codecov.yml b/.codecov.yml
@@ -0,0 +1,17 @@
+codecov:
+  branch: master
+
+comment: false
+
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 0%
+        only_pulls: false
+    patch:
+      default:
+        target: auto
+        threshold: 0%
+        only_pulls: true
diff --git a/.gitignore b/.gitignore
@@ -18,3 +18,6 @@ dist-newstyle
 postgrest.hp
 postgrest.prof
 __pycache__
+*.tix
+coverage
+.hpc
diff --git a/default.nix b/default.nix
@@ -66,7 +66,7 @@ let
 
   # Options passed to cabal in dev tools and tests
   devCabalOptions =
-    "-f FailOnWarn --test-show-detail=direct";
+    "-f dev --test-show-detail=direct";
 
   profiledHaskellPackages =
     pkgs.haskell.packages."${compiler}".extend (self: super:
@@ -87,7 +87,7 @@ rec {
   # libraries and documentation. We disable running the test suite on Nix
   # builds, as they require a database to be set up.
   postgrestPackage =
-    lib.dontCheck (lib.enableCabalFlag postgrest "FailOnWarn");
+    lib.dontCheck postgrest;
 
   # Static executable.
   postgrestStatic =
@@ -114,7 +114,11 @@ rec {
 
   # Scripts for running tests.
   tests =
-    pkgs.callPackage nix/tests.nix { inherit postgrest postgrestStatic postgrestProfiled postgresqlVersions devCabalOptions; };
+    pkgs.callPackage nix/tests.nix {
+      inherit postgrest postgrestProfiled postgresqlVersions devCabalOptions;
+      ghc = pkgs.haskell.compiler."${compiler}";
+      hpc-codecov = pkgs.haskell.packages."${compiler}".hpc-codecov;
+    };
 
   # Linting and styling scripts.
   style =

diff --git a/nix/devtools.nix b/nix/devtools.nix
@@ -72,6 +72,8 @@ let
       }
       ''
         ${cabal-install}/bin/cabal v2-clean
+        # clean old coverage data, too
+        rm -rf .hpc coverage
       '';
 
   check =

diff --git a/nix/tests.nix b/nix/tests.nix
@@ -5,15 +5,15 @@
 , checkedShellScript
 , curl
 , devCabalOptions
-, diffutils
+, ghc
+, gnugrep
 , haskell
+, hpc-codecov
 , lib
 , postgresql
 , postgresqlVersions
 , postgrest
 , postgrestProfiled
-, postgrestStatic
-, procps
 , python3
 , runtimeShell
 , yq
@@ -164,6 +164,90 @@ let
             postgrest --dump-schema \
             | ${yq}/bin/yq -y .
       '';
+
+  coverage =
+    name: postgresql:
+    checkedShellScript
+      {
+        inherit name;
+        docs = "Run spec and io tests while collecting hpc coverage data.";
+        inRootDir = true;
+      }
+      ''
+        env="$(cat ${postgrest.env})"
+        export PATH="$env/bin:$PATH"
+
+        # clean up previous coverage reports
+        mkdir -p coverage
+        rm -rf coverage/*
+
+        # temporary directory to collect data in
+        tmpdir="$(mktemp -d)"
+
+        # we keep the tmpdir when an error occurs for debugging and only remove it on success
+        trap 'echo Temporary directory kept at: $tmpdir' ERR SIGINT SIGTERM
+
+        # build once before running all the tests
+        ${cabal-install}/bin/cabal v2-build ${devCabalOptions} --enable-tests all
+
+        # collect all tests
+        HPCTIXFILE="$tmpdir"/io.tix \
+        ${withTmpDb postgresql} ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} \
+          ${ioTestPython}/bin/pytest -- -v test/io-tests
+          
+        HPCTIXFILE="$tmpdir"/spec.tix \
+        ${withTmpDb postgresql} ${cabal-install}/bin/cabal v2-test ${devCabalOptions}
+
+        # collect all the tix files
+        ${ghc}/bin/hpc sum  --union --exclude=Paths_postgrest --output="$tmpdir"/tests.tix "$tmpdir"/io.tix "$tmpdir"/spec.tix
+
+        # prepare the overlay
+        ${ghc}/bin/hpc overlay --output="$tmpdir"/overlay.tix test/coverage.overlay
+        ${ghc}/bin/hpc sum --union --output="$tmpdir"/tests-overlay.tix "$tmpdir"/tests.tix "$tmpdir"/overlay.tix
+
+        # check nothing in the overlay is actually tested
+        ${ghc}/bin/hpc map --function=inv --output="$tmpdir"/inverted.tix "$tmpdir"/tests.tix
+        ${ghc}/bin/hpc combine --function=sub \
+          --output="$tmpdir"/check.tix "$tmpdir"/overlay.tix "$tmpdir"/inverted.tix
+        # returns zero exit code if any count="<non-zero>" lines are found, i.e.
+        # something is covered by both the overlay and the tests
+        if ${ghc}/bin/hpc report --xml "$tmpdir"/check.tix | ${gnugrep}/bin/grep -qP 'count="[^0]'
+        then
+          ${ghc}/bin/hpc markup --highlight-covered --destdir=coverage/overlay "$tmpdir"/overlay.tix || true
+          ${ghc}/bin/hpc markup --highlight-covered --destdir=coverage/check "$tmpdir"/check.tix || true
+          echo "ERROR: Something is covered by both the tests and the overlay:"
+          echo "file://$(pwd)/coverage/check/hpc_index.html"
+          exit 1
+        else
+          # copy the result .tix file to the coverage/ dir to make it available to postgrest-coverage-draft-overlay, too
+          cp "$tmpdir"/tests-overlay.tix coverage/postgrest.tix
+          # prepare codecov json report
+          ${hpc-codecov}/bin/hpc-codecov --mix=.hpc --out=coverage/codecov.json coverage/postgrest.tix
+
+          # create html and stdout reports
+          # TODO: The markup command fails when run outside nix-shell (i.e. in CI!)
+          # Need to fix it properly in the future instead of adding the || true
+          ${ghc}/bin/hpc markup --destdir=coverage coverage/postgrest.tix || true
+          echo "file://$(pwd)/coverage/hpc_index.html"
+          ${ghc}/bin/hpc report coverage/postgrest.tix "$@"
+        fi
+
+        rm -rf "$tmpdir"
+      '';
+
+  coverageDraftOverlay =
+    name:
+    checkedShellScript
+      {
+        inherit name;
+        docs = "Create a draft overlay from current coverage report.";
+        inRootDir = true;
+      }
+      ''
+        ${ghc}/bin/hpc draft --output=test/coverage.overlay coverage/postgrest.tix
+        sed -i 's|^module \(.*\):|module \1/|g' test/coverage.overlay
+      '';
+
 in
 # Create an environment that contains all the utility scripts for running tests
   # that we defined above.
@@ -179,6 +263,8 @@ buildEnv
         testSpecAllVersions.bin
         (testIO "postgrest-test-io" postgresql).bin
         (dumpSchema "postgrest-dump-schema" postgresql).bin
+        (coverage "postgrest-coverage" postgresql).bin
+        (coverageDraftOverlay "postgrest-coverage-draft-overlay").bin
       ] ++ testSpecVersions;
   }
   # The memory tests have large dependencies (a profiled build of PostgREST)

diff --git a/postgrest.cabal b/postgrest.cabal
@@ -19,12 +19,17 @@ source-repository head
   type:     git
   location: git://github.com/PostgREST/postgrest.git
 
-flag FailOnWarn
+flag dev
   default:     False
   manual:      True
-  description: No warnings allowed
+  description: Development flags
 
 library
+  default-language:   Haskell2010
+  default-extensions: OverloadedStrings
+                      QuasiQuotes
+                      NoImplicitPrelude
+  hs-source-dirs:     src
   exposed-modules:    PostgREST.ApiRequest
                       PostgREST.App
                       PostgREST.Auth
@@ -43,7 +48,6 @@ library
                       PostgREST.Private.Common
                       PostgREST.Private.ProxyUri
                       PostgREST.Private.QueryFragment
-  hs-source-dirs:     src
   build-depends:      base                      >= 4.9 && < 4.15
                     , HTTP                      >= 4000.3.7 && < 4000.4
                     , Ranged-sets               >= 0.3 && < 0.5
@@ -88,15 +92,12 @@ library
                     , wai-extra                 >= 3.0.19 && < 3.2
                     , wai-logger                >= 2.3.2
                     , wai-middleware-static     >= 0.8.1 && < 0.10
-  default-language:   Haskell2010
-  default-extensions: OverloadedStrings
-                      QuasiQuotes
-                      NoImplicitPrelude
-  if flag(FailOnWarn)
-    ghc-options: -O2 -Werror -Wall -fwarn-identities
+  if flag(dev)
+    ghc-options: -O0 -Werror -Wall -fwarn-identities
                  -fno-spec-constr -optP-Wno-nonportable-include-path
+                 -fhpc -hpcdir .hpc
   else
-    ghc-options: -O2 -Wall -fwarn-identities
+    ghc-options: -O2 -Werror -Wall -fwarn-identities
                  -fno-spec-constr -optP-Wno-nonportable-include-path
           -- -fno-spec-constr may help keep compile time memory use in check,
           --   see https://gitlab.haskell.org/ghc/ghc/issues/16017#note_219304
@@ -105,8 +106,12 @@ library
           --   see https://github.com/commercialhaskell/stack/issues/3918
 
 executable postgrest
-  main-is:            Main.hs
+  default-language:   Haskell2010
+  default-extensions: OverloadedStrings
+                      QuasiQuotes
+                      NoImplicitPrelude
   hs-source-dirs:     main
+  main-is:            Main.hs
   build-depends:      base                >= 4.9 && < 4.15
                     , aeson               >= 1.4.7 && < 1.6
                     , auto-update         >= 0.1.4 && < 0.2
@@ -126,17 +131,14 @@ executable postgrest
                     , time                >= 1.6 && < 1.11
                     , wai                 >= 3.2.1 && < 3.3
                     , warp                >= 3.2.12 && < 3.4
-  default-language:   Haskell2010
-  default-extensions: OverloadedStrings
-                      QuasiQuotes
-                      NoImplicitPrelude
-  if flag(FailOnWarn)
+  if flag(dev)
     ghc-options:      -threaded -rtsopts "-with-rtsopts=-N -I2"
-                      -O2 -Werror -Wall -fwarn-identities
+                      -O0 -Werror -Wall -fwarn-identities
                       -fno-spec-constr -optP-Wno-nonportable-include-path
+                      -fhpc -hpcdir .hpc
   else
     ghc-options:      -threaded -rtsopts "-with-rtsopts=-N -I2"
-                      -O2 -Wall -fwarn-identities
+                      -O2 -Werror -Wall -fwarn-identities
                       -fno-spec-constr -optP-Wno-nonportable-include-path
 
   if !os(windows)
@@ -145,6 +147,11 @@ executable postgrest
 
 test-suite spec
   type:               exitcode-stdio-1.0
+  default-language:   Haskell2010
+  default-extensions: OverloadedStrings
+                      QuasiQuotes
+                      NoImplicitPrelude
+  hs-source-dirs:     test
   main-is:            Main.hs
   other-modules:      Feature.AndOrParamsSpec
                       Feature.AsymmetricJwtSpec
@@ -179,7 +186,6 @@ test-suite spec
                       Feature.UpsertSpec
                       SpecHelper
                       TestTypes
-  hs-source-dirs:     test
   build-depends:      base              >= 4.9 && < 4.15
                     , aeson             >= 1.4.7 && < 1.6
                     , aeson-qq          >= 0.8.1 && < 0.9
@@ -211,20 +217,21 @@ test-suite spec
                     , transformers-base >= 0.4.4 && < 0.5
                     , wai               >= 3.2.1 && < 3.3
                     , wai-extra         >= 3.0.19 && < 3.2
-  default-language:   Haskell2010
-  default-extensions: OverloadedStrings
-                      QuasiQuotes
-                      NoImplicitPrelude
   ghc-options:        -threaded -rtsopts -with-rtsopts=-N
+                      -O0 -Werror -Wall -fwarn-identities
+                      -fno-spec-constr -optP-Wno-nonportable-include-path
+                      -fno-warn-missing-signatures
 
-Test-Suite spec-querycost
-  Type:                exitcode-stdio-1.0
-  Default-Language:    Haskell2010
-  default-extensions:  OverloadedStrings, QuasiQuotes, NoImplicitPrelude
-  Hs-Source-Dirs:      test
-  Main-Is:             QueryCost.hs
-  Other-Modules:       SpecHelper
-  Build-Depends:       base              >= 4.9 && < 4.15
+test-suite spec-querycost
+  type:                exitcode-stdio-1.0
+  default-language:    Haskell2010
+  default-extensions:  OverloadedStrings
+                       QuasiQuotes
+                       NoImplicitPrelude
+  hs-source-dirs:      test
+  main-is:             QueryCost.hs
+  other-modules:       SpecHelper
+  build-depends:       base              >= 4.9 && < 4.15
                      , aeson             >= 1.4.7 && < 1.6
                      , aeson-qq          >= 0.8.1 && < 0.9
                      , async             >= 2.1.1 && < 2.3
@@ -256,3 +263,5 @@ Test-Suite spec-querycost
                      , transformers-base >= 0.4.4 && < 0.5
                      , wai               >= 3.2.1 && < 3.3
                      , wai-extra         >= 3.0.19 && < 3.2
+  ghc-options:         -O0 -Werror -Wall -fwarn-identities
+                       -fno-spec-constr -optP-Wno-nonportable-include-path
diff --git a/stack.yaml b/stack.yaml
@@ -1,9 +1,5 @@
 resolver: lts-16.26 # 2020-12-13, GHC 8.8.4
 
-flags:
-  postgrest:
-    FailOnWarn: true
-
 nix:
   packages:
     - pcre

diff --git a/test/Feature/InsertSpec.hs b/test/Feature/InsertSpec.hs
@@ -1,14 +1,10 @@
 module Feature.InsertSpec where
 
-import qualified Data.Aeson as JSON
-
 import Data.List              (lookup)
-import Data.Maybe             (fromJust)
 import Network.Wai            (Application)
 import Network.Wai.Test       (SResponse (simpleBody, simpleHeaders, simpleStatus))
 import Test.Hspec             hiding (pendingWith)
 import Test.Hspec.Wai.Matcher (bodyEquals)
-import TestTypes              (CompoundPK (..), IncPK (..))
 
 import Network.HTTP.Types
 import Test.Hspec.Wai
@@ -441,9 +437,6 @@ spec actualPgVersion = do
 
   describe "Row level permission" $
     it "set user_id when inserting rows" $ do
-      post "/postgrest/users" [json| { "id":"jdoe", "pass": "1234", "role": "postgrest_test_author" } |]
-      post "/postgrest/users" [json| { "id":"jroe", "pass": "1234", "role": "postgrest_test_author" } |]
-
       request methodPost "/authors_only"
           [ authHeaderJWT "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXN0X3Rlc3RfYXV0aG9yIiwiaWQiOiJqZG9lIn0.B-lReuGNDwAlU1GOC476MlO0vAt9JNoHIlxg2vwMaO0", ("Prefer", "return=representation") ]
           [json| { "secret": "nyancat" } |]

diff --git a/test/Feature/QueryLimitedSpec.hs b/test/Feature/QueryLimitedSpec.hs
@@ -1,7 +1,6 @@
 module Feature.QueryLimitedSpec where
 
-import Network.Wai      (Application)
-import Network.Wai.Test (SResponse (simpleHeaders, simpleStatus))
+import Network.Wai (Application)
 
 import Network.HTTP.Types
 import Test.Hspec

diff --git a/test/Feature/RollbackSpec.hs b/test/Feature/RollbackSpec.hs
@@ -7,8 +7,7 @@ import Test.Hspec
 import Test.Hspec.Wai
 import Test.Hspec.Wai.JSON
 
-import Protolude  hiding (get)
-import SpecHelper
+import Protolude hiding (get)
 
 -- two helpers functions to make sure that each test can setup and cleanup properly