Add postgrest-loadtest based on vegeta

.github/workflows/loadtest.yaml

@@ -0,0 +1,34 @@
+name: Loadtest
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - v*
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  Loadtest-Nix:
+    name: Loadtest (Nix)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          fetch-depth: 0
+      - name: Setup Nix Environment
+        uses: ./.github/actions/setup-nix
+        with:
+          tools: loadtest
+      - name: Run loadtest
+        run: |
+          postgrest-loadtest-against main
+          postgrest-loadtest-report > loadtest/loadtest.md
+      - name: Upload report
+        uses: actions/upload-artifact@v2.2.4
+        with:
+          name: loadtest.md
+          path: loadtest/loadtest.md
+          if-no-files-found: error

.gitignore

@@ -21,3 +21,4 @@ __pycache__
 *.tix
 coverage
 .hpc
+loadtest

default.nix

@@ -123,6 +123,10 @@ rec {
   docker =
     pkgs.callPackage nix/tools/docker { postgrest = postgrestStatic; };
 
+  # Load testing tools.
+  loadtest =
+    pkgs.callPackage nix/tools/loadtest.nix { inherit withTools; };
+
   # Script for running memory tests.
   memory =
     pkgs.callPackage nix/tools/memory.nix { inherit postgrestProfiled withTools; };
@@ -148,5 +152,5 @@ rec {
     };
 
   withTools =
-    pkgs.callPackage nix/tools/withTools.nix { inherit postgresqlVersions; };
+    pkgs.callPackage nix/tools/withTools.nix { inherit devCabalOptions postgresqlVersions postgrest; };
 }

nix/overlays/checked-shell-script/checked-shell-script.nix

@@ -14,19 +14,18 @@
 { name
 , docs
 , args ? [ ]
-, addCommandCompletion ? false
+, positionalCompletion ? ""
 , inRootDir ? false
 , redirectTixFiles ? true
 , withEnv ? null
 , withPath ? [ ]
 , withTmpDir ? false
 }: text:
 let
+  # square brackets are a pain to escape - if even possible. just don't use them...
+  escape = builtins.replaceStrings [ "\n" ] [ " \\n" ];
+
   argsTemplate =
-    let
-      # square brackets are a pain to escape - if even possible. just don't use them...
-      escapedDocs = builtins.replaceStrings [ "\n" ] [ " \\n" ] docs;
-    in
     writeTextFile {
       inherit name;
       destination = "/${name}.m4"; # destination is needed to have the proper basename for completion
@@ -37,7 +36,7 @@ let
           # stripping the /nix/store/... path for nicer display
           BASH_ARGV0="$(basename "$0")"
 
-          # ARG_HELP([${name}], [${escapedDocs}])
+          # ARG_HELP([${name}], [${escape docs}])
           ${lib.strings.concatMapStrings (arg: "# " + arg) args}
           # ARG_POSITIONAL_DOUBLEDASH()
           # ARG_DEFAULTS_POS()
@@ -65,8 +64,8 @@ let
         ${argbash}/bin/argbash --type completion --strip all ${argsTemplate}/${name}.m4 > $out
       ''
 
-      + lib.optionalString addCommandCompletion ''
-        sed 's/COMPREPLY.*compgen -o bashdefault .*$/_command/' -i $out
+      + lib.optionalString (positionalCompletion != "") ''
+        sed 's#COMPREPLY.*compgen -o bashdefault .*$#${escape positionalCompletion}#' -i $out
       ''
     );
 

nix/tools/devTools.nix

@@ -29,7 +29,7 @@ let
             "ARG_POSITIONAL_SINGLE([command], [Command to run])"
             "ARG_LEFTOVERS([command arguments])"
           ];
-        addCommandCompletion = true;
+        positionalCompletion = "_command";
         redirectTixFiles = false; # will be done by sub-command
         inRootDir = true;
       }
@@ -72,8 +72,8 @@ let
         inRootDir = true;
       }
       ''
-        ${withTools}/bin/postgrest-with-all ${tests}/bin/postgrest-test-spec
-        ${withTools}/bin/postgrest-with-all ${tests}/bin/postgrest-test-querycost
+        ${withTools.withPgAll} ${tests}/bin/postgrest-test-spec
+        ${withTools.withPgAll} ${tests}/bin/postgrest-test-querycost
         ${tests}/bin/postgrest-test-doctests
         ${tests}/bin/postgrest-test-spec-idempotence
         ${tests}/bin/postgrest-test-io

nix/tools/loadtest.nix

@@ -0,0 +1,174 @@
+{ buildToolbox
+, checkedShellScript
+, jq
+, python3Packages
+, vegeta
+, withTools
+, writers
+}:
+let
+  runner =
+    checkedShellScript
+      {
+        name = "postgrest-loadtest-runner";
+        docs = "Run vegeta. Assume PostgREST to be running.";
+        args = [
+          "ARG_LEFTOVERS([additional vegeta arguments])"
+          "ARG_USE_ENV([PGRST_SERVER_UNIX_SOCKET], [], [Unix socket to connect to running PostgREST instance])"
+        ];
+      }
+      ''
+        # ARG_USE_ENV only adds defaults or docs for environment variables
+        # We manually implement a required check here
+        # See also: https://github.com/matejak/argbash/issues/80
+        : "''${PGRST_SERVER_UNIX_SOCKET:?PGRST_SERVER_UNIX_SOCKET is required}"
+
+        ${vegeta}/bin/vegeta -cpus 1 attack \
+                                     -unix-socket "$PGRST_SERVER_UNIX_SOCKET" \
+                                     -max-workers 1 \
+                                     -workers 1 \
+                                     -rate 0 \
+                                     -duration 60s \
+                                     "''${_arg_leftovers[@]}"
+      '';
+
+  loadtest =
+    checkedShellScript
+      {
+        name = "postgrest-loadtest";
+        docs = "Run the vegeta loadtests with PostgREST.";
+        args = [
+          "ARG_OPTIONAL_SINGLE([output], [o], [Filename to dump json output to], [./loadtest/result.bin])"
+          "ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/loadtest])"
+          "ARG_LEFTOVERS([additional vegeta arguments])"
+        ];
+        inRootDir = true;
+      }
+      ''
+        export PGRST_DB_CONFIG="false"
+        export PGRST_DB_POOL="1"
+        export PGRST_DB_TX_END="rollback-allow-override"
+        export PGRST_LOG_LEVEL="crit"
+
+        mkdir -p "$(dirname "$_arg_output")"
+
+        # shellcheck disable=SC2145
+        ${withTools.withPg} --fixtures "$_arg_testdir"/fixtures.sql \
+        ${withTools.withPgrst} \
+        sh -c "cd \"$_arg_testdir\" && ${runner} -targets targets.http \"''${_arg_leftovers[@]}\"" \
+          | tee "$_arg_output" \
+          | ${vegeta}/bin/vegeta report -type=text
+      '';
+
+  loadtestAgainst =
+    let
+      name = "postgrest-loadtest-against";
+    in
+    checkedShellScript
+      {
+        inherit name;
+        docs =
+          ''
+            Run the vegeta loadtest twice:
+            - once on the <target> branch
+            - once in the current worktree
+          '';
+        args = [
+          "ARG_POSITIONAL_SINGLE([target], [Commit-ish reference to compare with])"
+          "ARG_LEFTOVERS([additional vegeta arguments])"
+        ];
+        positionalCompletion =
+          ''
+            if test "$prev" == "${name}"; then
+              __gitcomp_nl "$(__git_refs)"
+            fi
+          '';
+        inRootDir = true;
+      }
+      ''
+        cat << EOF
+
+        Running loadtest on "$_arg_target"...
+
+        EOF
+
+        # Runs the test files from the current working tree
+        # to make sure both tests are run with the same files.
+        # Save the results in the current working tree, too,
+        # otherwise they'd be lost in the temporary working tree
+        # created by withTools.withGit.
+        ${withTools.withGit} "$_arg_target" ${loadtest} --output "$PWD/loadtest/$_arg_target.bin" --testdir "$PWD/test/loadtest" "''${_arg_leftovers[@]}"
+
+        cat << EOF
+
+        Done running on "$_arg_target".
+
+        EOF
+
+        cat << EOF
+
+        Running loadtest on HEAD...
+
+        EOF
+
+        ${loadtest} --output "$PWD/loadtest/head.bin" --testdir "$PWD/test/loadtest" "''${_arg_leftovers[@]}"
+
+        cat << EOF
+
+        Done running on HEAD.
+
+        EOF
+      '';
+
+  reporter =
+    checkedShellScript
+      {
+        name = "postgrest-loadtest-reporter";
+        docs = "Create a named json report for a single result file.";
+        args = [
+          "ARG_POSITIONAL_SINGLE([file], [Filename of result to create report for])"
+          "ARG_LEFTOVERS([additional vegeta arguments])"
+        ];
+        inRootDir = true;
+      }
+      ''
+        ${vegeta}/bin/vegeta report -type=json "$_arg_file" \
+          | ${jq}/bin/jq --arg branch "$(basename "$_arg_file" .bin)" '. + {branch: $branch}'
+      '';
+
+  toMarkdown =
+    writers.writePython3 "postgrest-loadtest-to-markdown"
+      {
+        libraries = [ python3Packages.pandas python3Packages.tabulate ];
+      }
+      ''
+        import sys
+        import pandas as pd
+
+        pd.read_json(sys.stdin) \
+          .set_index('param') \
+          .drop(['branch', 'earliest', 'end', 'latest']) \
+          .convert_dtypes() \
+          .to_markdown(sys.stdout, floatfmt='.0f')
+      '';
+
+
+  report =
+    checkedShellScript
+      {
+        name = "postgrest-loadtest-report";
+        docs = "Create a report of all loadtest reports as markdown.";
+        inRootDir = true;
+      }
+      ''
+        find loadtest -type f -iname '*.bin' -exec ${reporter} {} \; \
+          | ${jq}/bin/jq '[leaf_paths as $path | {param: $path | join("."), (.branch): getpath($path)}]' \
+          | ${jq}/bin/jq --slurp 'flatten | group_by(.param) | map(add)' \
+          | ${toMarkdown}
+      '';
+
+in
+buildToolbox {
+  name = "postgrest-loadtest";
+  tools = [ loadtest loadtestAgainst report ];
+}

nix/tools/memory.nix

@@ -17,7 +17,7 @@ let
         withPath = [ postgrestProfiled curl ];
       }
       ''
-        ${withTools.latest} test/memory-tests.sh
+        ${withTools.withPg} test/memory-tests.sh
       '';
 
 in

nix/tools/tests.nix

@@ -24,7 +24,7 @@ let
         withEnv = postgrest.env;
       }
       ''
-        ${withTools.latest} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec
+        ${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec
       '';
 
   testQuerycost =
@@ -36,7 +36,7 @@ let
         withEnv = postgrest.env;
       }
       ''
-        ${withTools.latest} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:querycost
+        ${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:querycost
       '';
 
   testDoctests =
@@ -66,7 +66,7 @@ let
         withEnv = postgrest.env;
       }
       ''
-        ${withTools.latest} ${runtimeShell} -c " \
+        ${withTools.withPg} ${runtimeShell} -c " \
           ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec && \
           ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec"
       '';
@@ -92,7 +92,7 @@ let
       }
       ''
         ${cabal-install}/bin/cabal v2-build ${devCabalOptions}
-        ${cabal-install}/bin/cabal v2-exec ${withTools.latest} \
+        ${cabal-install}/bin/cabal v2-exec ${withTools.withPg} \
           ${ioTestPython}/bin/pytest -- -v test/io-tests "''${_arg_leftovers[@]}"
       '';
 
@@ -106,7 +106,7 @@ let
         withPath = [ jq ];
       }
       ''
-        ${withTools.latest} \
+        ${withTools.withPg} \
             ${cabal-install}/bin/cabal v2-run ${devCabalOptions} --verbose=0 -- \
             postgrest --dump-schema \
             | ${yq}/bin/yq -y .
@@ -135,14 +135,14 @@ let
 
         # collect all tests
         HPCTIXFILE="$tmpdir"/io.tix \
-          ${withTools.latest} ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} \
+          ${withTools.withPg} ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} \
           ${ioTestPython}/bin/pytest -- -v test/io-tests
 
         HPCTIXFILE="$tmpdir"/spec.tix \
-          ${withTools.latest} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec
+          ${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec
 
         HPCTIXFILE="$tmpdir"/querycost.tix \
-          ${withTools.latest} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:querycost
+          ${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:querycost
 
         # Note: No coverage for doctests, as doctests leverage GHCi and GHCi does not support hpc