feat(api-headless-cms): pluginable crud operations (#1544) (#1570)

api/code/graphql/package.json

@@ -10,6 +10,7 @@
     "@webiny/api-file-manager-s3": "^5.5.0-beta.0",
     "@webiny/api-form-builder": "^5.5.0-beta.0",
     "@webiny/api-headless-cms": "^5.5.0-beta.0",
+    "@webiny/api-headless-cms-ddb-es": "^5.6.0",
     "@webiny/api-i18n": "^5.5.0-beta.0",
     "@webiny/api-i18n-content": "^5.5.0-beta.0",
     "@webiny/api-page-builder": "^5.5.0-beta.0",

api/code/graphql/src/index.ts

@@ -16,6 +16,7 @@ import fileManagerS3 from "@webiny/api-file-manager-s3";
 import formBuilderPlugins from "@webiny/api-form-builder/plugins";
 import securityPlugins from "./security";
 import headlessCmsPlugins from "@webiny/api-headless-cms/plugins";
+import cmsDynamoDbElasticsearch from "@webiny/api-headless-cms-ddb-es";
 
 const debug = process.env.DEBUG === "true";
 
@@ -51,7 +52,8 @@ export const handler = createHandler({
         }),
         pageBuilderPlugins(),
         formBuilderPlugins(),
-        headlessCmsPlugins()
+        headlessCmsPlugins(),
+        cmsDynamoDbElasticsearch()
     ],
     http: { debug }
 });

api/code/headlessCMS/package.json

@@ -7,6 +7,7 @@
   },
   "dependencies": {
     "@webiny/api-headless-cms": "^5.5.0-beta.0",
+    "@webiny/api-headless-cms-ddb-es": "^5.6.0",
     "@webiny/api-i18n": "^5.5.0-beta.0",
     "@webiny/api-i18n-content": "^5.5.0-beta.0",
     "@webiny/api-plugin-elastic-search-client": "^5.5.0-beta.0",

api/code/headlessCMS/src/index.ts

@@ -6,6 +6,7 @@ import dbPlugins from "@webiny/handler-db";
 import { DynamoDbDriver } from "@webiny/db-dynamodb";
 import elasticSearch from "@webiny/api-plugin-elastic-search-client";
 import headlessCmsPlugins from "@webiny/api-headless-cms/content";
+import cmsDynamoDbElasticsearch from "@webiny/api-headless-cms-ddb-es";
 import securityPlugins from "./security";
 import logsPlugins from "@webiny/handler-logs";
 
@@ -27,7 +28,8 @@ export const handler = createHandler({
         securityPlugins(),
         i18nPlugins(),
         i18nContentPlugins(),
-        headlessCmsPlugins({ debug })
+        headlessCmsPlugins({ debug }),
+        cmsDynamoDbElasticsearch()
     ],
     http: { debug }
 });

jest.config.base.js

@@ -29,6 +29,9 @@ module.exports = ({ path }, presets = []) => {
                 babelConfig: `${path}/.babelrc.js`,
                 diagnostics: false
             }
-        }
+        },
+        collectCoverage: false,
+        collectCoverageFrom: ["packages/**/*.{ts,tsx,js,jsx}"],
+        coverageReporters: ["html"]
     });
 };

jest.config.js

@@ -2,15 +2,132 @@ const fs = require("fs");
 const path = require("path");
 const { allWorkspaces } = require("@webiny/project-utils/workspaces");
 
+const createPackageJestConfigPath = pkg => {
+    const jestConfigPath = path.join(pkg, "jest.config.js");
+    if (!fs.existsSync(jestConfigPath)) {
+        return null;
+    }
+    return jestConfigPath;
+};
+const createPackageJestSetupPath = pkg => {
+    const setupPath = path.join(pkg, "jest.setup.js");
+    if (!fs.existsSync(setupPath)) {
+        return null;
+    }
+    return setupPath;
+};
+
+const getPackageKeywords = pkg => {
+    const file = path.join(pkg, "package.json");
+    if (!fs.existsSync(file)) {
+        throw new Error(`Missing package.json "${file}".`);
+    }
+    const content = fs.readFileSync(file).toString();
+    try {
+        const json = JSON.parse(content);
+        return Array.isArray(json.keywords) ? json.keywords : [];
+    } catch (ex) {
+        throw new Error(`Could not parse package.json "${file}".`);
+    }
+};
+
+const hasPackageJestConfig = pkg => {
+    return !!createPackageJestConfigPath(pkg);
+};
+
+const getPackageJestSetup = pkg => {
+    const setupPath = createPackageJestSetupPath(pkg);
+    if (!setupPath) {
+        return null;
+    }
+    return require(setupPath);
+};
+
+const identifiers = {};
+const createPackageName = initialName => {
+    let name = initialName;
+    let current = 0;
+    while (identifiers[name]) {
+        name = `${initialName}-${current}`;
+    }
+    return name;
+};
+
+const createPackageFilter = (args = []) => {
+    const filters = args
+        .filter(arg => {
+            return arg.startsWith("--keyword=");
+        })
+        .map(arg => {
+            return arg.replace("--keyword=", "");
+        });
+    return (packageKeywords = []) => {
+        if (
+            !packageKeywords ||
+            filters.length === 0 ||
+            (packageKeywords.length === 0 && filters.length === 0)
+        ) {
+            return true;
+        }
+        for (const filter of filters) {
+            // a single keyword in the argument
+            const result = packageKeywords.includes(filter);
+            if (result) {
+                return true;
+            }
+        }
+        return false;
+    };
+};
+
+const isPackageAllowed = createPackageFilter(process.argv);
+
 const projects = allWorkspaces()
-    .map(pkg => {
-        if (!fs.existsSync(path.join(pkg, "jest.config.js"))) {
-            return null;
+    .reduce((collection, pkg) => {
+        const hasConfig = hasPackageJestConfig(pkg);
+        const setup = getPackageJestSetup(pkg);
+        const basePackagePath = pkg.replace(process.cwd() + "/", "");
+
+        if (!hasConfig && !setup) {
+            return collection;
+        }
+        // we need to filter out the packages that do not match required keywords, if any
+        const keywords = getPackageKeywords(pkg);
+        if (!isPackageAllowed(keywords)) {
+            return collection;
+        }
+
+        if (setup && (Array.isArray(setup) === true || setup["0"] !== undefined)) {
+            for (const key in setup) {
+                if (!setup.hasOwnProperty(key)) {
+                    continue;
+                }
+                const subPackage = setup[key];
+                // we need to filter out the subpackage as well
+                if (!isPackageAllowed(subPackage.keywords)) {
+                    continue;
+                }
+                // keywords does not exist in jest config so remove it
+                // there will be error if not removed
+                delete subPackage["keywords"];
+                const name = createPackageName(subPackage.name);
+                collection.push({
+                    ...subPackage,
+                    name: name,
+                    displayName: name,
+                    rootDir: subPackage.rootDir || pkg
+                });
+            }
+            return collection;
         }
-        return pkg.replace(process.cwd() + "/", "");
-    })
+        return collection.concat([basePackagePath]);
+    }, [])
     .filter(Boolean);
 
+if (projects.length === 0) {
+    console.log(`There are no packages found. Please check the filters if you are using those.`);
+    process.exit(1);
+}
 module.exports = {
     projects,
     modulePathIgnorePatterns: ["dist"],

packages/api-headless-cms-ddb-es/.babelrc.js

@@ -0,0 +1 @@
+module.exports = require("../../.babel.node")({ path: __dirname });

packages/api-headless-cms-ddb-es/.typedoc.json

@@ -0,0 +1,4 @@
+{
+  "name": "API Headless CMS DynamoDB / Elasticsearch",
+  "entryPoints": ["types.ts"]
+}

packages/api-headless-cms-ddb-es/CHANGELOG.md

@@ -0,0 +1,4 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.

packages/api-headless-cms-ddb-es/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Webiny
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/api-headless-cms-ddb-es/README.md

@@ -0,0 +1,18 @@
+# @webiny/api-headless-cms-ddb-es
+
+[![](https://img.shields.io/npm/dw/@webiny/api-headless-cms-ddb-es.svg)](https://www.npmjs.com/package/@webiny/api-headless-cms-ddb-es)
+[![](https://img.shields.io/npm/v/@webiny/api-headless-cms-ddb-es.svg)](https://www.npmjs.com/package/@webiny/api-headless-cms-ddb-es)
+[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier)
+[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com)
+
+## Install
+
+```
+npm install --save @webiny/api-headless-cms-ddb-es
+```
+
+Or if you prefer yarn:
+
+```
+yarn add @webiny/api-headless-cms-ddb-es
+```

packages/api-headless-cms/__tests__/README.md → packages/api-headless-cms-ddb-es/__tests__/__api__/README.md

@@ -1,4 +1,4 @@
-# Tests
+# api-headless-cms
 
 ### Env variables
 
@@ -10,5 +10,5 @@ Custom port for local elasticsearch.
 
 ##### Usage
 ````
-ELASTICSEARCH_PORT=9200 LOCAL_ELASTICSEARCH=true yarn test packages/api-headless-cms
+ELASTICSEARCH_PORT=9200 LOCAL_ELASTICSEARCH=true yarn test packages/api-headless-cms --keyword=cms:ddb-es --keyword=cms:base
 ````

packages/api-headless-cms-ddb-es/__tests__/__api__/environment.js

@@ -0,0 +1,124 @@
+const dbPlugins = require("@webiny/handler-db").default;
+const { DynamoDbDriver } = require("@webiny/db-dynamodb");
+const { DocumentClient } = require("aws-sdk/clients/dynamodb");
+const elasticSearch = require("@webiny/api-plugin-elastic-search-client").default;
+const { createHandler } = require("@webiny/handler-aws");
+const dynamoToElastic = require("@webiny/api-dynamodb-to-elasticsearch/handler").default;
+const { Client } = require("@elastic/elasticsearch");
+const { simulateStream } = require("@webiny/project-utils/testing/dynamodb");
+const NodeEnvironment = require("jest-environment-node");
+/**
+ * For this to work it must load plugins that have already been built
+ */
+const plugins = require("../../dist/index").default;
+
+if (typeof plugins !== "function") {
+    throw new Error(`Loaded plugins file must export a function that returns an array of plugins.`);
+}
+
+const ELASTICSEARCH_PORT = process.env.ELASTICSEARCH_PORT || "9200";
+
+const getStorageOperationsPlugins = ({
+    elasticsearchClient,
+    documentClient,
+    elasticSearchContext
+}) => {
+    // Intercept DocumentClient operations and trigger dynamoToElastic function (almost like a DynamoDB Stream trigger)
+    simulateStream(documentClient, createHandler(elasticSearchContext, dynamoToElastic()));
+
+    return () => {
+        return [
+            plugins(),
+            dbPlugins({
+                table: "HeadlessCms",
+                driver: new DynamoDbDriver({
+                    documentClient
+                })
+            }),
+            elasticSearchContext,
+            {
+                type: "context",
+                async apply() {
+                    await elasticsearchClient.indices.putTemplate({
+                        name: "headless-cms-entries-index",
+                        body: {
+                            index_patterns: ["*headless-cms*"],
+                            settings: {
+                                analysis: {
+                                    analyzer: {
+                                        lowercase_analyzer: {
+                                            type: "custom",
+                                            filter: ["lowercase", "trim"],
+                                            tokenizer: "keyword"
+                                        }
+                                    }
+                                }
+                            },
+                            mappings: {
+                                properties: {
+                                    property: {
+                                        type: "text",
+                                        fields: {
+                                            keyword: {
+                                                type: "keyword",
+                                                ignore_above: 256
+                                            }
+                                        },
+                                        analyzer: "lowercase_analyzer"
+                                    },
+                                    rawValues: {
+                                        type: "object",
+                                        enabled: false
+                                    }
+                                }
+                            }
+                        }
+                    });
+                }
+            }
+        ];
+    };
+};
+
+class CmsTestEnvironment extends NodeEnvironment {
+    async setup() {
+        await super.setup();
+
+        const elasticsearchClient = new Client({
+            node: `http://localhost:${ELASTICSEARCH_PORT}`
+        });
+        const documentClient = new DocumentClient({
+            convertEmptyValues: true,
+            endpoint: process.env.MOCK_DYNAMODB_ENDPOINT || "http://localhost:8001",
+            sslEnabled: false,
+            region: "local",
+            accessKeyId: "test",
+            secretAccessKey: "test"
+        });
+        const elasticSearchContext = elasticSearch({
+            endpoint: `http://localhost:${ELASTICSEARCH_PORT}`,
+            auth: {}
+        });
+        const clearEsIndices = async () => {
+            return elasticsearchClient.indices.delete({
+                index: "_all"
+            });
+        };
+        /**
+         * This is a global function that will be called inside the tests to get all relevant plugins, methods and objects.
+         */
+        this.global.__getStorageOperationsPlugins = () => {
+            return getStorageOperationsPlugins({
+                elasticsearchClient,
+                elasticSearchContext,
+                documentClient
+            });
+        };
+        this.global.__beforeEach = clearEsIndices;
+        this.global.__afterEach = clearEsIndices;
+        this.global.__beforeAll = clearEsIndices;
+        this.global.__afterAll = clearEsIndices;
+    }
+}
+
+module.exports = CmsTestEnvironment;