Add graphql pub syncing

README.md

@@ -79,6 +79,24 @@ function query(
 
 An asynchronous function which returns a GraphQL response promise for a given query, variables, and context.
 
+```ts
+const QUERY = `{
+  workspaces {
+    address
+    documents {
+      ... on ES3Document {
+        value
+        author {
+          address
+        }
+      }
+    }
+  }  
+}`;
+
+const result = query(QUERY, {}, context);
+```
+
 #### `schema`
 
 The GraphQLSchema which you can use to create things like HTTP servers:
@@ -94,3 +112,18 @@ server.listen().then(({ url }) => {
   console.log(`🍄 earthstar-graphql ready at ${url}`);
 });
 ```
+
+#### `syncGraphql`
+
+```ts
+function syncGraphql(
+  storage: IStorage,
+  graphqlUrl: string,
+  filters: {
+    pathPrefix?: string;
+    versionsByAuthor?: string;
+  }
+): void;
+```
+
+A function you can use to sync documents from a remote GraphQL server to a local `IStorage` instance. It also takes filter options, so you can selectively sync documents. The `IStorage` is mutated in place.

package.json

@@ -3,6 +3,7 @@
   "license": "AGPL-3.0",
   "version": "2.0.0",
   "dependencies": {
+    "cross-fetch": "^3.0.5",
     "earthstar": "3.1.1",
     "graphql": "15.3.0",
     "graphql-type-json": "0.3.2",
@@ -19,16 +20,18 @@
     "test": "jest",
     "build": "tsdx build",
     "prepare": "tsdx build",
+    "dev": "ts-node-dev --respawn --no-notify --transpile-only -P server.tsconfig.json ./src/scripts/start-server.ts --ignore=built",
     "server": "ts-node --transpile-only -P server.tsconfig.json ./src/scripts/start-server.ts --ignore=built"
   },
   "devDependencies": {
-    "apollo-server": "2.16.0",
     "@babel/preset-env": "^7.10.4",
     "@babel/preset-typescript": "^7.10.4",
     "@types/jest": "^26.0.5",
     "@types/js-base64": "^3.0.0",
     "@types/node": "^14.0.24",
+    "apollo-server": "2.16.0",
     "jest": "^26.1.0",
+    "msw": "^0.20.1",
     "prettier": "^2.0.5",
     "ts-node": "^8.10.2",
     "ts-node-dev": "^1.0.0-pre.52",

src/create-schema-context.ts

@@ -0,0 +1,77 @@
+import { ValidatorEs3, StorageSqlite, StorageMemory } from "earthstar";
+import {
+  Context,
+  AddWorkspaceCheck,
+  SqliteContext,
+  MemoryContext,
+  StorageType,
+  SyncFiltersArg,
+} from "./types";
+
+export const VALIDATORS = [ValidatorEs3];
+
+type CommonContextOptions = {
+  workspaceAddresses: string[];
+  canAddWorkspace?: AddWorkspaceCheck;
+  syncFilters?: SyncFiltersArg;
+};
+
+type SqliteContextOptions = {
+  getWorkspacePath: (address: string) => string;
+} & CommonContextOptions;
+
+type ContextOptions = CommonContextOptions | SqliteContextOptions;
+
+const EMPTY_SYNC_FILTERS = {
+  pathPrefixes: [],
+  versionsByAuthors: [],
+};
+
+export default function createSchemaContext(
+  mode: "MEMORY",
+  options: CommonContextOptions
+): MemoryContext;
+export default function createSchemaContext(
+  mode: "SQLITE",
+  options: SqliteContextOptions
+): SqliteContext;
+export default function createSchemaContext(
+  mode: StorageType,
+  options: ContextOptions
+): Context {
+  if (mode === "MEMORY") {
+    return {
+      storageMode: "MEMORY",
+      workspaces: options.workspaceAddresses.map(
+        (addr) => new StorageMemory(VALIDATORS, addr)
+      ),
+      canAddWorkspace: options.canAddWorkspace
+        ? options.canAddWorkspace
+        : () => true,
+      syncFilters: {
+        ...EMPTY_SYNC_FILTERS,
+        ...options.syncFilters,
+      },
+    };
+  }
+
+  return {
+    storageMode: "SQLITE",
+    workspaces: options.workspaceAddresses.map((addr) => {
+      return new StorageSqlite({
+        mode: "create-or-open",
+        validators: VALIDATORS,
+        filename: (options as SqliteContextOptions).getWorkspacePath(addr),
+        workspace: addr,
+      });
+    }),
+    canAddWorkspace: options.canAddWorkspace
+      ? options.canAddWorkspace
+      : () => true,
+    getWorkspacePath: (options as SqliteContextOptions).getWorkspacePath,
+    syncFilters: {
+      ...EMPTY_SYNC_FILTERS,
+      ...options.syncFilters,
+    },
+  };
+}

src/index.ts

@@ -1,3 +1,4 @@
 export { default as query } from "./query";
 export { default as schema } from "./schema";
-export { makeMemoryContext, makeSqliteContext } from "./context";
+export { default as createSchemaContext } from "./create-schema-context";
+export { default as syncGraphql } from "./sync-graphql";

src/mocks/create-mock-server.ts

@@ -0,0 +1,38 @@
+import { generateAuthorKeypair, sign, Document, ValidatorEs3 } from "earthstar";
+import { createSchemaContext, query } from "..";
+import { graphql } from "msw";
+import { PULL_QUERY, INGEST_MUTATION } from "../sync-graphql";
+import { setupServer } from "msw/node";
+import { GraphQLError } from "graphql";
+import { Context } from "../types";
+
+function createHandlers(context: Context) {
+  return [
+    graphql.query("PullQuery", async (req, res, ctx) => {
+      const { data, errors } = await query(PULL_QUERY, req.variables, context);
+
+      if (errors) {
+        return res(ctx.errors(errors as Partial<GraphQLError>[]));
+      }
+
+      return res(ctx.data(data));
+    }),
+    graphql.mutation("IngestMutation", async (req, res, ctx) => {
+      const { data, errors } = await query(
+        INGEST_MUTATION,
+        req.variables,
+        context
+      );
+
+      if (errors) {
+        return res(ctx.errors(errors as Partial<GraphQLError>[]));
+      }
+
+      return res(ctx.data(data));
+    }),
+  ];
+}
+
+export default function createMockServer(context: Context) {
+  return setupServer(...createHandlers(context));
+}

src/query.test.ts

@@ -1,10 +1,12 @@
 import query from "./query";
-import { makeMemoryContext } from "./context";
+import createSchemaContext from "./create-schema-context";
 import { generateAuthorKeypair, sign, verify } from "earthstar";
 
 describe("query", () => {
   test("Successfully queries a new context", async () => {
-    var ctx = makeMemoryContext(["+testing.123"]);
+    const ctx = createSchemaContext("MEMORY", {
+      workspaceAddresses: ["+testing.123"],
+    });
 
     const TEST_QUERY = `query TestQuery {
           workspaces {
@@ -16,20 +18,24 @@ describe("query", () => {
     const result = await query(TEST_QUERY, {}, ctx);
 
     expect(result).toEqual({
-      data: { workspaces: [{ address: "+testing.123", name: "testing" }] },
+      data: {
+        workspaces: [{ address: "+testing.123", name: "testing" }],
+      },
     });
   });
 
   test("Can modify a context's data with a mutation query", async () => {
-    var ctx = makeMemoryContext(["+testing.123"]);
+    const ctx = createSchemaContext("MEMORY", {
+      workspaceAddresses: ["+testing.123"],
+    });
 
     const variables = {
       author: generateAuthorKeypair("test"),
       document: { path: "/testing", value: "is fun!" },
       workspace: "+testing.123",
     };
 
-    const TEST_MUTATION = `mutation TestMutation($author: AuthorInput!, $document: DocumentInput!, $workspace: String!) {
+    const TEST_MUTATION = `mutation TestMutation($author: AuthorInput!, $document: NewDocumentInput!, $workspace: String!) {
           set(
               author: $author,
               document: $document,
@@ -51,7 +57,9 @@ describe("query", () => {
   });
 
   test("Can add a workspace with a mutation query", async () => {
-    var ctx = makeMemoryContext(["+testing.123"]);
+    const ctx = createSchemaContext("MEMORY", {
+      workspaceAddresses: ["+testing.123"],
+    });
 
     const newAddress = "+newspace.123";
 
@@ -80,16 +88,19 @@ describe("query", () => {
     const permittedKeypair = generateAuthorKeypair("test");
     const message = sign(permittedKeypair, "canAddWorkspace");
 
-    const restrictedCtx = makeMemoryContext([], (address, keypair) => {
-      if (
-        address === "+allowed.999" &&
-        keypair &&
-        verify(keypair.address, message, "canAddWorkspace")
-      ) {
-        return true;
-      }
+    const restrictedCtx = createSchemaContext("MEMORY", {
+      workspaceAddresses: [],
+      canAddWorkspace: (address, keypair) => {
+        if (
+          address === "+allowed.999" &&
+          keypair &&
+          verify(keypair.address, message, "canAddWorkspace")
+        ) {
+          return true;
+        }
 
-      return false;
+        return false;
+      },
     });
 
     const badAuthorVars = {

src/scripts/start-server.ts

@@ -1,4 +1,4 @@
-import { makeSqliteContext } from "../context";
+import createSchemaContext from "../create-schema-context";
 import fs from "fs";
 import path from "path";
 import { ApolloServer } from "apollo-server";
@@ -30,15 +30,16 @@ const workspaces =
     })
     .filter(isDefined) || [];
 
-makeSqliteContext(workspaces, (addr) => `./workspaces/${addr}.sqlite`).then(
-  (context) => {
-    const server = new ApolloServer({ schema, context });
-    server
-      .listen({
-        port: 4000,
-      })
-      .then(({ url }) => {
-        console.log(`🍄 earthstar-graphql ready at ${url}`);
-      });
-  }
-);
+const context = createSchemaContext("SQLITE", {
+  workspaceAddresses: workspaces,
+  getWorkspacePath: (addr) => path.resolve(`./workspaces/${addr}.sqlite`),
+});
+
+const server = new ApolloServer({ schema, context });
+server
+  .listen({
+    port: 4000,
+  })
+  .then(({ url }) => {
+    console.log(`🍄 earthstar-graphql ready at ${url}`);
+  });