Rewrite GraphQL schema generation and query parsing (close #2801)

.circleci/server-upgrade-downgrade/err_msg.patch

@@ -0,0 +1,13 @@
+diff --git a/server/tests-py/validate.py b/server/tests-py/validate.py
+index 3eecd52a..a18b3113 100644
+--- a/server/tests-py/validate.py
++++ b/server/tests-py/validate.py
+@@ -318,7 +318,7 @@ def assert_graphql_resp_expected(resp_orig, exp_response_orig, query, resp_hdrs=
+             # If it is a batch GraphQL query, compare each individual response separately
+             for (exp, out) in zip(as_list(exp_response), as_list(resp)):
+                 matched_ = equal_CommentedMap(exp, out)
+-                if is_err_msg(exp):
++                if is_err_msg(exp) and is_err_msg(out):
+                     if not matched_:
+                         warnings.warn("Response does not have the expected error message\n" + dump_str.getvalue())
+                         return resp, matched

.circleci/server-upgrade-downgrade/run-dev.sh

@@ -6,7 +6,9 @@
 # and sets some of the required variables that run.sh needs,
 # before executing run.sh
 set -euo pipefail
-ROOT="${BASH_SOURCE[0]%/*}"
+cd "${BASH_SOURCE[0]%/*}"
+ROOT="${PWD}"
+cd - > /dev/null
 
 SERVER_DIR="$ROOT/../../server"
 
@@ -18,8 +20,8 @@ echo "server binary: $SERVER_BINARY"
 cd -
 set +x
 
-export SERVER_OUTPUT_DIR="server-output"
-export LATEST_SERVER_BINARY="./graphql-engine-latest"
+export SERVER_OUTPUT_DIR="$ROOT/server-output"
+export LATEST_SERVER_BINARY="$ROOT/graphql-engine-latest"
 
 # Create Python virtualenv
 if ! [ -f ".venv/bin/activate" ] ; then
@@ -40,7 +42,8 @@ log_duration=on
 port=$PG_PORT
 EOF
 )
-export HASURA_GRAPHQL_DATABASE_URL="postgres://postgres:$PGPASSWORD@127.0.0.1:$PG_PORT/postgres"
+# Pytest is giving out deprecated warnings when postgres:// is used
+export HASURA_GRAPHQL_DATABASE_URL="postgresql://postgres:$PGPASSWORD@127.0.0.1:$PG_PORT/postgres"
 
 DOCKER_PSQL="docker exec -u postgres -it $PG_CONTAINER_NAME psql -p $PG_PORT"
 

.circleci/server-upgrade-downgrade/run.sh

@@ -12,7 +12,9 @@ set -euo pipefail
 # # echo an error message before exiting
 # trap 'echo "\"${last_command}\" command filed with exit code $?."' EXIT
 
-ROOT="${BASH_SOURCE[0]%/*}"
+cd "${BASH_SOURCE[0]%/*}"
+ROOT="${PWD}"
+cd - > /dev/null
 
 download_with_etag_check() {
 	URL="$1"
@@ -119,6 +121,17 @@ trap rm_worktree ERR
 
 make_latest_release_worktree() {
 	git worktree add --detach "$WORKTREE_DIR" "$RELEASE_VERSION"
+        cd "$WORKTREE_DIR"
+        # FIX ME: Remove the patch below after the next stable release
+        # The --avoid-error-message-checks in pytest was implementated  as a rather relaxed check than
+        # what we intended to have. In versions <= v1.3.0,
+        # this check allows response to be success even if the expected response is a failure.
+        # The patch below fixes that issue.
+        # The `git apply` should give errors from next release onwards,
+        # since this change is going to be included in the next release version
+        git apply "${ROOT}/err_msg.patch" || \
+		(log "Remove the git apply in make_latest_release_worktree function"  && false)
+        cd - > /dev/null
 }
 
 cleanup_hasura_metadata_if_present() {
@@ -148,7 +161,18 @@ get_server_upgrade_tests() {
 	cd $RELEASE_PYTEST_DIR
 	tmpfile="$(mktemp --dry-run)"
 	set -x
-	python3 -m pytest -q --collect-only --collect-upgrade-tests-to-file "$tmpfile" -m 'allow_server_upgrade_test and not skip_server_upgrade_test' "${args[@]}" 1>/dev/null 2>/dev/null
+	# FIX ME: Deselecting some introspection tests from the previous test suite
+	# which throw errors on the latest build. Even when the output of the current build is more accurate.
+	# Remove these deselects after the next stable release
+	python3 -m pytest -q --collect-only --collect-upgrade-tests-to-file "$tmpfile" \
+		-m 'allow_server_upgrade_test and not skip_server_upgrade_test' \
+		--deselect test_schema_stitching.py::TestRemoteSchemaBasic::test_introspection \
+		--deselect test_schema_stitching.py::TestAddRemoteSchemaCompareRootQueryFields::test_schema_check_arg_default_values_and_field_and_arg_types \
+                --deselect test_graphql_mutations.py::TestGraphqlInsertPermission::test_user_with_no_backend_privilege \
+                --deselect test_graphql_mutations.py::TestGraphqlInsertPermission::test_backend_user_no_admin_secret_fail \
+                --deselect test_graphql_mutations.py::TestGraphqlMutationCustomSchema::test_update_article \
+                --deselect test_graphql_queries.py::TestGraphQLQueryEnums::test_introspect_user_role \
+		"${args[@]}" 1>/dev/null 2>/dev/null
 	set +x
 	cat "$tmpfile"
 	cd - >/dev/null
@@ -174,11 +198,12 @@ run_server_upgrade_pytest() {
 		set -x
 
 		# With --avoid-error-message-checks, we are only going to throw warnings if the error message has changed between releases
- 		# FIX ME: Remove the deselect below after the next stable release
 		pytest --hge-urls "${HGE_URL}"  --pg-urls "$HASURA_GRAPHQL_DATABASE_URL" \
 			--avoid-error-message-checks "$@" \
 			-m 'allow_server_upgrade_test and not skip_server_upgrade_test' \
-			--deselect test_graphql_mutations.py::TestGraphqlUpdateBasic::test_numerics_inc \
+                        --deselect test_graphql_mutations.py::TestGraphqlInsertPermission::test_user_with_no_backend_privilege \
+                        --deselect test_graphql_mutations.py::TestGraphqlMutationCustomSchema::test_update_article \
+                        --deselect test_graphql_queries.py::TestGraphQLQueryEnums::test_introspect_user_role \
 			-v $tests_to_run
 		set +x
 		cd -

.circleci/test-server.sh

@@ -191,9 +191,7 @@ pip3 install -r requirements.txt
 # node js deps
 curl -sL https://deb.nodesource.com/setup_8.x | bash -
 apt-get install -y nodejs
-npm_config_loglevel=error npm install $PYTEST_ROOT/remote_schemas/nodejs/
-
-npm install apollo-server graphql
+(cd remote_schemas/nodejs && npm_config_loglevel=error npm ci)
 
 mkdir -p "$OUTPUT_FOLDER/hpc"
 

.kodiak.toml

@@ -1,7 +1,7 @@
 ###############################################################################
 ## Configuration for auto-merge / auto-update bot
 ##
-## See: https://kodiakhq.com/ 
+## See: https://kodiakhq.com/
 ###############################################################################
 
 # Kodiak's configuration file should be placed at `.kodiak.toml` (repository

CHANGELOG.md

@@ -2,19 +2,31 @@
 
 ## Next release
 
+### Breaking changes
+
+This release contains the [PDV refactor (#4111)](https://github.com/hasura/graphql-engine/pull/4111), a significant rewrite of the internals of the server, which did include some breaking changes:
+
+- The semantics of explicit `null` values in `where` filters have changed according to the discussion in [issue 704](https://github.com/hasura/graphql-engine/issues/704#issuecomment-635571407): an explicit `null` value in a comparison input object will be treated as an error rather than resulting in the expression being evaluated to `True`. For instance: `delete_users(where: {id: {_eq: $userId}}) { name }` will yield an error if `$userId` is `null` instead of deleting all users.
+- The validation of required headers has been fixed (closing #14 and #3659):
+  - if a query selects table `bar` through table `foo` via a relationship, the required permissions headers will be the union of the required headers of table `foo` and table `bar` (we used to only check the headers of the root table);
+  - if an insert does not have an `on_conflict` clause, it will not require the update permissions headers.
+
 
 ### Bug fixes and improvements
 
 (Add entries here in the order of: server, console, cli, docs, others)
 
-- docs: add docs page on networking with docker (close #4346) (#4811)
+- server: some mutations that cannot be performed will no longer be in the schema (for instance, `delete_by_pk` mutations won't be shown to users that do not have select permissions on all primary keys) (#4111)
+- server: miscellaneous description changes (#4111)
+- server: treat the absence of `backend_only` configuration and `backend_only: false` equally (closing #5059) (#4111)
 - cli: add missing global flags for seeds command (#5565)
+- docs: add docs page on networking with docker (close #4346) (#4811)
 
 ## `v1.3.1`, `v1.3.1-beta.1`
 
 ### Breaking change
 
-Headers from environment variables starting with `HASURA_GRAPHQL_` are not allowed  
+Headers from environment variables starting with `HASURA_GRAPHQL_` are not allowed
 in event triggers, actions & remote schemas.
 
 If you do have such headers configured, then you must update the header configuration before upgrading.

server/.stylish-haskell.yaml

@@ -229,6 +229,7 @@ language_extensions:
 - BangPatterns
 - BlockArguments
 - ConstraintKinds
+- DataKinds
 - DefaultSignatures
 - DeriveDataTypeable
 - DeriveFoldable
@@ -238,9 +239,11 @@ language_extensions:
 - DeriveTraversable
 - DerivingVia
 - EmptyCase
+- ExistentialQuantification
 - FlexibleContexts
 - FlexibleInstances
 - FunctionalDependencies
+- GADTs
 - GeneralizedNewtypeDeriving
 - InstanceSigs
 - LambdaCase
@@ -249,12 +252,16 @@ language_extensions:
 - NamedFieldPuns
 - NoImplicitPrelude
 - OverloadedStrings
+- QuantifiedConstraints
 - QuasiQuotes
 - RankNTypes
+- RecordWildCards
+- RoleAnnotations
 - ScopedTypeVariables
 - StandaloneDeriving
 - TemplateHaskell
 - TupleSections
 - TypeApplications
 - TypeFamilies
+- TypeFamilyDependencies
 - TypeOperators

server/CONTRIBUTING.md

@@ -24,6 +24,11 @@ Additionally, you will need a way to run a Postgres database server. The `dev.sh
 - [PostgreSQL](https://www.postgresql.org) >= 9.5
 - [postgis](https://postgis.net)
 
+Additionally, you will need a way to run a Postgres database server. The `dev.sh` script (described below) can set up a Postgres instance for you via [Docker](https://www.docker.com), but if you want to run it yourself, you’ll need:
+
+- [PostgreSQL](https://www.postgresql.org) >= 9.5
+- [postgis](https://postgis.net)
+
 ### Upgrading npm
 
 If your npm is too old (>= 5.7 required):
@@ -116,16 +121,13 @@ cabal new-run -- test:graphql-engine-tests \
 
 ##### Running the Python test suite
 
-1. To run the Python tests, you’ll need to install the necessary Python dependencies first. It is
-   recommended that you do this in a self-contained Python venv, which is supported by Python 3.3+
-   out of the box. To create one, run:
+1. To run the Python tests, you’ll need to install the necessary Python dependencies first. It is recommended that you do this in a self-contained Python venv, which is supported by Python 3.3+ out of the box. To create one, run:
 
    ```
    python3 -m venv .python-venv
    ```
 
-   (The second argument names a directory where the venv sandbox will be created; it can be anything
-   you like, but `.python-venv` is `.gitignore`d.)
+   (The second argument names a directory where the venv sandbox will be created; it can be anything you like, but `.python-venv` is `.gitignore`d.)
 
    With the venv created, you can enter into it in your current shell session by running:
 
@@ -141,19 +143,26 @@ cabal new-run -- test:graphql-engine-tests \
    pip3 install -r tests-py/requirements.txt
    ```
 
-3. Start an instance of `graphql-engine` for the test suite to use:
+3. Install the dependencies for the Node server used by the remote schema tests:
+
+   ```
+   (cd tests-py/remote_schemas/nodejs && npm ci)
+   ```
+
+4. Start an instance of `graphql-engine` for the test suite to use:
 
    ```
    env EVENT_WEBHOOK_HEADER=MyEnvValue \
        WEBHOOK_FROM_ENV=http://localhost:5592/ \
+       SCHEDULED_TRIGGERS_WEBHOOK_DOMAIN=http://127.0.0.1:5594 \
      cabal new-run -- exe:graphql-engine \
        --database-url='postgres://<user>:<password>@<host>:<port>/<dbname>' \
        serve --stringify-numeric-types
    ```
 
    The environment variables are needed for a couple tests, and the `--stringify-numeric-types` option is used to avoid the need to do floating-point comparisons.
 
-4. With the server running, run the test suite:
+5. With the server running, run the test suite:
 
    ```
    cd tests-py

server/cabal.project

@@ -15,10 +15,17 @@
 packages: .
 
 constraints:
-  -- ensure we don’t end up with a freeze file that forces an incompatible
+  -- ensure we don't end up with a freeze file that forces an incompatible
   -- version in CI for Setup.hs scripts.
   setup.Cabal <3.4
 
+allow-newer:
+  -- dependent-map depends on constraints-extras, but its bounds have not yet
+  -- been relaxed for GHC 8.10.
+  constraints-extras-0.3.0.2:base,
+  constraints-extras-0.3.0.2:constraints,
+  constraints-extras-0.3.0.2:template-haskell
+
 package *
   optimization: 2
   -- For tooling, e.g. 'weeder', and IDE-like stuff:
@@ -41,7 +48,7 @@ source-repository-package
 source-repository-package
   type: git
   location: https://github.com/hasura/graphql-parser-hs.git
-  tag: f4a093981ca5626982a17c2bfaad047cc0834a81
+  tag: 8f1cd3a9bf6ec91f1ba1d83f704ab078113e035b
 
 source-repository-package
   type: git

server/cabal.project.ci

@@ -3,6 +3,12 @@
 reject-unconstrained-dependencies: all
 
 package graphql-engine
-  ghc-options: -j3 -Werror
+  ghc-options:
+    -j3 -Werror
+    -- Limit heap size to 8GB, which is the amount of available memory on a
+    -- CircleCI `large` instance. (GHC interprets G as GB, i.e. 1,000^3 bytes,
+    -- but instances seem to have 8GiB, i.e. 1,024^3 bytes, so that leaves us
+    -- a little breathing room.)
+    +RTS -M8G -RTS
   tests: true
   benchmarks: true

server/cabal.project.dev

@@ -6,13 +6,15 @@
 -- Or, if you want to customize the configuration:
 --   $ cp cabal.project.dev cabal.project.local
 
+with-compiler: ghc-8.10.1
+
 package *
   documentation: true
 
 package graphql-engine
   -- NOTE: this seems to work so long as there is no 'ghc-options: -O2' in the cabal file,
-  -- but new-build will report 'Build profile: -O1' for some reason. 
-  -- See:https://github.com/haskell/cabal/issues/6221 
+  -- but new-build will report 'Build profile: -O1' for some reason.
+  -- See:https://github.com/haskell/cabal/issues/6221
   optimization: 0
   documentation: false
   flags: +developer

server/cabal.project.freeze

@@ -79,6 +79,8 @@ constraints: any.Cabal ==3.2.0.0,
              any.conduit ==1.3.2,
              any.connection ==0.3.1,
              any.constraints ==0.12,
+             any.constraints-extras ==0.3.0.2,
+             constraints-extras +build-readme,
              any.containers ==0.6.2.1,
              any.contravariant ==1.5.2,
              contravariant +semigroups +statevar +tagged,
@@ -112,8 +114,8 @@ constraints: any.Cabal ==3.2.0.0,
              any.deepseq ==1.4.4.0,
              any.deferred-folds ==0.9.10.1,
              any.dense-linear-algebra ==0.1.0.0,
-             any.dependent-map ==0.2.4.0,
-             any.dependent-sum ==0.4,
+             any.dependent-map ==0.4.0.0,
+             any.dependent-sum ==0.7.1.0,
              any.directory ==1.3.6.1,
              any.distributive ==0.6.2,
              distributive +semigroups +tagged,
@@ -127,7 +129,6 @@ constraints: any.Cabal ==3.2.0.0,
              any.erf ==2.0.0.0,
              any.errors ==2.3.0,
              any.exceptions ==0.10.4,
-             exceptions +transformers-0-4,
              any.fail ==4.9.0.0,
              any.fast-logger ==3.0.1,
              any.file-embed ==0.0.11.2,
@@ -242,8 +243,6 @@ constraints: any.Cabal ==3.2.0.0,
              any.primitive-unlifted ==0.1.3.0,
              any.process ==1.6.8.2,
              any.profunctors ==5.5.2,
-             any.protolude ==0.3.0,
-             protolude -dev,
              any.psqueues ==0.2.7.2,
              any.quickcheck-instances ==0.3.22,
              quickcheck-instances -bytestring-builder,
@@ -277,6 +276,8 @@ constraints: any.Cabal ==3.2.0.0,
              any.simple-sendfile ==0.2.30,
              simple-sendfile +allow-bsd,
              any.socks ==0.6.1,
+             any.some ==1.0.1,
+             some +newtype-unsafe,
              any.split ==0.2.3.4,
              any.splitmix ==0.0.4,
              splitmix -optimised-mixer +random,

server/commit_diff.txt

@@ -0,0 +1 @@
+**** Latest commit compared against master - fd7fb580831fe9054164a285441c99562f34c815