Merge pull request #333 from walmartlabs/20200901-federation

Add support for implementing Apollo GraphQL federation services

CHANGES.md

@@ -2,6 +2,8 @@
 
 Optional request tracing is now designed to be compatible with Apollo GraphQL's implementation.
 
+New support for [Apollo GraphQL Federation](https://www.apollographql.com/docs/apollo-server/federation/introduction/).
+
 ## 0.37.0 -- 30 Jun 2020
 
 Added new function `com.walmartlabs.lacinia.util/inject-streamers`, used to
@@ -15,7 +17,7 @@ to `com.walmartlabs.lacinia.parser.schema/parse-schema` which is now deprecated.
 The schema parser has been updated, to allow input values (within input types)
 to specify defaults, and to support extending input types.
 
-The preview API (`selections-seq2`, etc.) now recognize the `@include` and
+The preview API functions (`selections-seq2`, etc.) now recognize the `@include` and
 `@skip` directives.
 
 Exceptions thrown by resolver functions are now caught and wrapped as new exceptions

dev-resources/no-entities-federation.sdl

@@ -0,0 +1,8 @@
+type User  {
+    id: Int!
+    name: String
+}
+
+# Just want a union to show that _entities is not present
+
+union Stuff = User

dev-resources/simple-federation.sdl

@@ -0,0 +1,20 @@
+type User @key(fields: "id") {
+    id: Int!
+    name: String!
+}
+
+type Query {
+    user_by_id(id: Int!) : User
+}
+
+schema { query: Query }
+
+type Account @key(fields: "acct_number") {
+  acct_number: String!
+  name: String!
+}
+
+type Product @key(fields: "upc") @extends {
+  upc: String! @external
+  reviewed_by: User
+}

docs/_examples/fed/external.gql

@@ -0,0 +1,16 @@
+type User @extends @key(fields: "id") {
+  id: String! @external
+  favoriteProducts: [Product]
+}
+
+type Product @key(fields: "upc") {
+  upc: String!
+  name: String!
+  price: Int!
+}
+
+type Query {
+    productByUpc(upc: String!) : Product
+}
+
+schema { query: Query }

docs/_examples/fed/internal.gql

@@ -0,0 +1,10 @@
+type User @key(fields: "id") {
+  id: String!
+  name: String!
+}
+
+type Query {
+  userById(id:String!) : User
+}
+
+schema { query: Query }

docs/_examples/fed/products.edn

@@ -0,0 +1,58 @@
+(ns products.server
+  (:require
+   [io.pedestal.http :as http]
+   [clojure.java.io :as io]
+   [com.walmartlabs.lacinia.pedestal2 :as lp]
+   [com.walmartlabs.lacinia.parser.schema :refer [parse-schema]]
+   [com.walmartlabs.lacinia.schema :as schema]
+   [com.walmartlabs.lacinia.util :as util]))
+
+(defn resolve-users-external
+  [_ _ reps]
+  (for [{:keys [id]} reps]
+    (schema/tag-with-type {:id id} 
+                          :User)))
+
+(defn get-product-by-upc
+  [context upc]
+  ;; Peform DB query here, return map with :upc, :name, :price
+  )
+
+(defn get-favorite-products-for-user
+  [context user-id]
+  ;; Perform DB query here, return seq of maps with :upc, :name, :price
+  )
+
+(defn resolve-products-internal
+  [context _ reps]
+  (for [{:keys [upc]} reps
+        :let [product (get-product-by-upc context upc)]]
+    (schema/tag-with-type product :Product)))
+
+(defn resolve-product-by-upc
+  [context {:keys [upc]} _]
+  (get-product-by-upc context upc))
+
+(defn resolve-favorite-products
+  [context _ user]
+  (let [{:keys [id]} user]
+    (get-favorite-products-for-user context id)))
+
+(defn products-schema
+  []
+  (-> "products.gql"
+      io/resource
+      slurp
+      (parse-schema {:federation {:entity-resolvers {:Product resolve-products-internal
+                                                     :User resolve-users-external}}})
+      (util/inject-resolvers {:Query/productByUpc #'resolve-product-by-upc
+                              :User/favoriteProducts #'resolve-favorite-products})
+      schema/compile))
+
+(defn start
+  []
+  (-> (products-schema)
+      lp/default-service
+      http/create-server
+      http/start))
+

docs/_examples/fed/query.gql

@@ -0,0 +1,6 @@
+{ 
+    userById(id: "124c41") { 
+        name 
+        favoriteProducts { upc name price } 
+    } 
+}

docs/_examples/fed/schema.gql

@@ -0,0 +1,9 @@
+scalar _Any
+scalar _FieldSet
+
+# a union of all types that use the @key directive
+union _Entity
+
+extend type Query {
+  _entities(representations: [_Any!]!): [_Entity]!
+}

docs/conf.py

@@ -76,7 +76,7 @@
 
 # General information about the project.
 project = u'com.walmartlabs/lacinia'
-copyright = u'2015-2018, Walmartlabs'
+copyright = u'2015-2020, Walmartlabs'
 author = u'Lacinia Contributors'
 
 # The version info for the project you're documenting, acts as replacement for

docs/federation/external.rst

@@ -0,0 +1,39 @@
+External Entities
+=================
+
+The previous example showed an internal entity that can be extended; this example shows a different service providing
+its own internal entity, but also extending the ``User`` entity.
+
+.. literalinclude:: /_examples/fed/external.gql
+
+Note the use of the ``@extends`` directive, this indicates that ``User`` (in the products service) is a stub for the full
+``User`` entity in the users service.
+
+You must ensure that the external ``User`` includes the same ``@key`` directive (or directives), and the same primary key
+fields; here ``id`` must be present, since it is part of the primary key.
+The ``@external`` directive indicates that the field is provided by another service (the users service).
+
+The ``favoriteProducts`` field on ``User`` is an addition provided by this service, the products service.
+Like any other field, a resolver must be provided for it.
+We'll see how that works shortly.
+
+Notice that this service adds the ``productByUpc`` query to the ``Query`` object; the Apollo GraphQL gateway
+merges together all the queries defined by all the implementing services.
+
+Again, the point of the gateway is that it mixes and matches from all the implementing services; clients should
+*only* go through the gateway since that's the only place where this merged view of all the individual schemas
+exists.
+
+The gateway is capable of building a plan that involves multiple steps to satisfy a client query.
+
+For example, consider this gateway query:
+
+.. literalinclude:: /_examples/fed/query.gql
+
+The gateway will start with a query to the users service; it will invoke the ``userById`` query there and will
+select both the ``name`` field (as specified in the client query) and the ``id`` field (since that's specified
+in the ``@key`` directive on the ``User`` entity).
+
+A second query will be sent to the products service.
+This query is used to get those favorite products; but to understand exactly
+how that works, we must first discuss `representations`.

docs/federation/implementation.rst

@@ -0,0 +1,49 @@
+Service Implementation
+======================
+
+At this point, we've discussed what goes into each implementing service's schema, and a bit about how
+each service is responsible for resolving representations; let's finally see how this all fits together with
+Lacinia.
+
+Below is a sketch of how this comes together in the products service:
+
+.. literalinclude:: /_examples/fed/products.edn
+
+The ``resolve-users-external`` function is used to convert a seq of ``User`` representations
+into a seq of ``User`` entity stubs; this is called from the resolver for the ``_entities`` query whose type
+is a list of the ``_Entities`` union, therefore each value :doc:`must be tagged </resolve/type-tags>` with the ``:User`` type.
+
+.. sidebar:: Return type
+
+    Return type here is just like a normal field resolver that returns GraphQL list; it may be seq of values, or a
+    :doc:`ResolverResult </resolve/resolve-as>` that delivers such a seq.
+
+``resolve-products-internal`` does the same for ``Product`` representations, but since this is the
+products service, the expected behavior is to perform a query against an external data store and
+ensure the results match the structure of the ``Product`` entity.
+
+``resolve-product-by-upc`` is the resolver function for the ``productByUpc`` query.
+Since the field type is ``Product`` there's no need to tag the value.
+
+``resolve-favorite-products`` is the resolver function for the ``User/favoriteProducts`` field.
+This is passed the ``User`` (provided by ``resolve-users-external``); it extracts the ``id`` and passes
+it to ``get-favorite-products-for-user``.
+
+The remainder is bare-bones scaffolding to read, parse, and compile the schema and build a Pedestal
+service endpoint around it.
+
+Pay careful attention to the call to :api:`parser-schema/parse-schema`; the presence of the
+``:federation`` option is critical; this adds the necessary base types and directives
+before parsing the schema definition, and then adds the ``_entities`` query and ``_Entities`` union
+afterwards, among other things.
+
+The ``:entity-resolvers`` map is critical; this maps from a type name to a entity resolver;
+this information is used to build the field resolver function for the ``_entities`` query.
+
+.. warning::
+
+    A lot of details are left out of this, such as initializing the database and storing
+    the database connection into the application context, where functions like
+    ``get-product-by-upc`` can access it.  
+
+    This is only a sketch to help you connect the dots.

docs/federation/index.rst

@@ -0,0 +1,63 @@
+Apollo GraphQL Federation
+=========================
+
+GraphQL federation is a concept, spearheaded by
+`Apollo GraphQL <https://www.apollographql.com/docs/apollo-server/federation/introduction/>`_, a Node-based JavaScript project,
+whereby multiple GraphQL schemas can be combined, behind a single `gateway` service.
+It's a useful concept, as it allows different teams to stand up their own schemas and services, written in
+any language, and dynamically combine them into a single "super" schema.
+
+Each service's schema can evolve independentently (as long as that evolution is backwards compatible), and can deploy
+on its own cycle. The gateway becomes the primary entrypoint for all clients, and it knows how to break
+service-spanning queries apart and build a overall query plan.
+
+Lacinia has been extended, starting in 0.38.0, to support acting as an implementing service; there is no plan
+at this time to act as a gateway.
+
+.. warning::
+
+    At this time, only a schema defined with the :doc:`Schema Definition Language </schema/parsing>`, can be extended to act as
+    a service implementation.
+
+Essentially, federation allows a set of services to each provide their own types, queries, mutations and organizes things so that
+each service can provide additional fields to the types provided by the other services.
+
+The `Apollo GraphQL documentation <https://www.apollographql.com/docs/apollo-server/federation/introduction/#concern-based-separation>`_ includes
+a basic example, where a users service exposes a ``User`` type (and related queries), a products service exposes
+a ``Product`` type, and a reviews service exposes a ``Review`` type.
+
+Without federation, these individual services are useful, but limited.
+A smart client could know about all three services, and send a series of requests to each, to build 
+up a model of, say, a particular user and the products that user has reviewed.
+
+For example:
+
+* Query the users service for the user, providing the user's unique id
+* Query the reviews service to get a list of reviews for that specific user (again, passing the user's unique id)
+* Query the products service for details (name, price, etc.) for each product reviewed by the user
+
+... but this is a lot to heap on the client developers; each client will have to manage three sets of GraphQL endpoints, and know exactly
+which fields are needed to bridge relationships between the different services.
+
+Instead, federation allows the Apollo GraphQL gateway to merge together the three individual services into one composite service.
+The client is only needs access to the single gateway enpoint, and is free to make complex queries that
+span from ``User`` to ``Review`` to ``Product`` seamlessly;
+the gateway service is responsible for communicating to the implementing services.
+
+A GraphQL type (or interface) that can span services this way is called an `entity`.
+
+In federation terms, the ``User`` entity is `internal` to the users service, and `external` to the other two services.
+The users service defines all the fields of the ``User`` entity, and can add new fields whenever necessary while staying
+backwards compatible, just as with a traditional GraphQL schema.
+
+In the other schemas, the ``User`` type is `external`; just a kind of stub for ``User`` is defined in the schemas for
+the products and reviews service. The full type, and the stub, must agree on fields that define the primary key
+for the entity. 
+
+.. toctree::
+    :hidden:
+
+    internal
+    external
+    reps
+    implementation

docs/federation/internal.rst

@@ -0,0 +1,29 @@
+Internal Entities
+=================
+
+Defining an entity that is internal is quite straight-forward, it is almost the same as in
+traditional GraphQL.
+
+.. sidebar:: Examples
+
+   Here, and in the remaining examples, we've simplified the example from
+   Apollo GraphQL's documentation to just two services:
+   users and products.
+
+.. literalinclude:: /_examples/fed/internal.gql
+
+When federation is enabled, a 
+`number of new directives <https://www.apollographql.com/docs/apollo-server/federation/federation-spec/#key>`_ are automatically available, 
+including ``@key``, which defines the primary key, or primary keys, for the entity.
+
+The above example would be the schema for a users service that can be extended by other services.
+
+.. warning::
+
+   Lacinia's default name for the object containing queries is ``QueryRoot``, whereas the default for Apollo and 
+   `most other GraphQL libraries <https://graphql.org/learn/schema/#the-query-and-mutation-types>`_ is ``Query``.  
+
+   Because of this, and because `Apollo doesn't do the right thing here <https://github.com/apollographql/apollo-server/issues/4554>`_,
+   it is necessary to override Lacinia's default by adding ``schema { query : Query }``.
+
+   The same holds true for mutations, which must be on a type named ``Mutation``.

docs/federation/reps.rst

@@ -0,0 +1,46 @@
+Representations
+===============
+
+A `representation` is a map that can be transferred from one implementing service to another, within the same federation.
+This is necessary to allow work started in one service to continue in another; consider the query:
+
+.. literalinclude:: /_examples/fed/query.gql
+
+The gateway will query the ``User/favoriteProducts`` field on the products service as the second step on this query ... but
+where does the ``User`` come from?
+
+After the gateway performs the initial query on the users service, it builds a representation of the specific ``User``
+to pass to the products service, using information from the ``@key`` directive:
+
+.. code-block:: json
+
+    {"__typename": "User",
+     "id": "124c41"}
+
+This representation is JSON, and is be passed to an implementing service's ``_entities`` query, which is automaticaly added
+to the implementing service's schema by Lacinia:
+
+.. literalinclude:: /_examples/fed/schema.gql
+
+The ``_Entity`` union will contain all entities, internal or external, in the local schema; for the products service, this
+will be ``User`` and ``Product``.
+
+The ``_entities`` query exists to convert some number of representations (here, as scalar type ``_Any``) into entities
+(either stub entities or full entities).
+The gateway sends a request that passes the representations in, and uses fragments to extract the data needed
+by the original client query::
+
+    query($representations:[_Any!]!) {
+        _entities(representations:$representations) {
+            ... on User {
+                favoriteProducts {upc name price}
+            }
+        }
+    }
+
+So, in the products service, the ``_entities`` resolver converts the representation into a stub ``User`` object,
+containing just enough information so that the ``favoriteProducts`` resolver can perform whatever database query
+it uses.
+The response from the products service is merged together with the response from the users service and a final
+response can be returned to the gateway service's client.
+