Merge pull request #12918 from strapi/chore/knex-v1

Make strapi compatible with knex v1

.github/actions/run-e2e-tests/script.sh

@@ -7,5 +7,5 @@ export ENV_PATH="$(pwd)/testApp/.env"
 
 opts=($DB_OPTIONS)
 
-yarn run -s test:generate-app "${opts[@]}" $@
-yarn run -s test:e2e
+yarn run -s test:generate-app "${opts[@]}"
+yarn run -s test:e2e $@

.github/workflows/tests.yml

@@ -15,9 +15,10 @@ jobs:
         node: [12, 14, 16]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2-beta
+      - uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node }}
+          cache: yarn
       - uses: ./.github/actions/install-modules
       - name: Run lint
         run: yarn run -s lint
@@ -33,9 +34,10 @@ jobs:
         node: [12, 14, 16]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2-beta
+      - uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node }}
+          cache: yarn
       - uses: ./.github/actions/install-modules
         with:
           globalPackages: codecov
@@ -53,9 +55,10 @@ jobs:
         node: [12, 14, 16]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2-beta
+      - uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node }}
+          cache: yarn
       - uses: ./.github/actions/install-modules
         with:
           globalPackages: codecov
@@ -71,7 +74,6 @@ jobs:
     strategy:
       matrix:
         node: [12, 14, 16]
-      max-parallel: 3
     services:
       postgres:
         # Docker Hub image
@@ -92,9 +94,10 @@ jobs:
           - 5432:5432
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2-beta
+      - uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node }}
+          cache: yarn
       - uses: ./.github/actions/install-modules
       - uses: ./.github/actions/run-e2e-tests
         with:
@@ -107,7 +110,6 @@ jobs:
     strategy:
       matrix:
         node: [12, 14, 16]
-      max-parallel: 3
     services:
       mysql:
         image: mysql
@@ -143,7 +145,6 @@ jobs:
     strategy:
       matrix:
         node: [12, 14, 16]
-      max-parallel: 3
     services:
       mysql:
         image: mysql:5
@@ -175,11 +176,11 @@ jobs:
   e2e_ce_sqlite:
     runs-on: ubuntu-latest
     needs: [lint, unit_back, unit_front]
-    name: '[CE] E2E (sqlite, node: ${{ matrix.node }})'
+    name: '[CE] E2E (sqlite: ${{ matrix.sqlite_pkg }}, node: ${{ matrix.node }})'
     strategy:
       matrix:
         node: [12, 14, 16]
-      max-parallel: 3
+        sqlite_pkg: ['better-sqlite3', 'sqlite3', '@vscode/sqlite3']
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
@@ -188,9 +189,10 @@ jobs:
           cache: yarn
       - uses: ./.github/actions/install-modules
       - uses: ./.github/actions/run-e2e-tests
+        env:
+          SQLITE_PKG: ${{ matrix.sqlite_pkg }}
         with:
-          dbOptions: '--dbclient=sqlite --dbfile=./tmp/data.db'
-
+          dbOptions: '--dbclient=sqlite-legacy --dbfile=./tmp/data.db'
   # EE
   e2e_ee_pg:
     runs-on: ubuntu-latest
@@ -202,7 +204,6 @@ jobs:
     strategy:
       matrix:
         node: [12, 14, 16]
-      max-parallel: 3
     services:
       postgres:
         # Docker Hub image
@@ -243,7 +244,6 @@ jobs:
     strategy:
       matrix:
         node: [12, 14, 16]
-      max-parallel: 3
     services:
       mysql:
         image: mysql
@@ -276,14 +276,14 @@ jobs:
   e2e_ee_sqlite:
     runs-on: ubuntu-latest
     needs: [lint, unit_back, unit_front]
-    name: '[EE] E2E (sqlite, node: ${{ matrix.node }})'
+    name: '[EE] E2E (sqlite: ${{ matrix.sqlite_pkg }}, node: ${{ matrix.node }})'
     if: github.event.pull_request.head.repo.full_name == github.repository && !(github.actor == 'dependabot[bot]' || github.actor == 'dependabot-preview[bot]')
     env:
       STRAPI_LICENSE: ${{ secrets.strapiLicense }}
     strategy:
       matrix:
         node: [12, 14, 16]
-      max-parallel: 3
+        sqlite_pkg: ['better-sqlite3', 'sqlite3', '@vscode/sqlite3']
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
@@ -292,6 +292,8 @@ jobs:
           cache: yarn
       - uses: ./.github/actions/install-modules
       - uses: ./.github/actions/run-e2e-tests
+        env:
+          SQLITE_PKG: ${{ matrix.sqlite_pkg }}
         with:
           dbOptions: '--dbclient=sqlite --dbfile=./tmp/data.db'
           runEE: true

examples/getstarted/package.json

@@ -27,6 +27,8 @@
     "mysql": "2.18.1",
     "passport-google-oauth2": "0.2.0",
     "pg": "8.6.0",
+    "better-sqlite3": "7.5.0",
+    "@vscode/sqlite3": "5.0.7",
     "sqlite3": "5.0.2"
   },
   "strapi": {

packages/core/admin/server/bootstrap.js

@@ -12,12 +12,12 @@ const defaultAdminAuthSettings = {
   },
 };
 
-const registerPermissionActions = () => {
-  getService('permission').actionProvider.registerMany(adminActions.actions);
+const registerPermissionActions = async () => {
+  await getService('permission').actionProvider.registerMany(adminActions.actions);
 };
 
-const registerAdminConditions = () => {
-  getService('permission').conditionProvider.registerMany(adminConditions.conditions);
+const registerAdminConditions = async () => {
+  await getService('permission').conditionProvider.registerMany(adminConditions.conditions);
 };
 
 const registerModelHooks = () => {
@@ -53,8 +53,8 @@ const syncAuthSettings = async () => {
 };
 
 module.exports = async () => {
-  registerAdminConditions();
-  registerPermissionActions();
+  await registerAdminConditions();
+  await registerPermissionActions();
   registerModelHooks();
 
   const permissionService = getService('permission');

packages/core/database/lib/connection.js

@@ -0,0 +1,64 @@
+/* eslint-disable node/no-missing-require */
+/* eslint-disable node/no-extraneous-require */
+'use strict';
+
+const knex = require('knex');
+
+const SqliteClient = require('knex/lib/dialects/sqlite3/index');
+
+const trySqlitePackage = packageName => {
+  try {
+    require.resolve(packageName);
+    return packageName;
+  } catch (error) {
+    if (error.code === 'MODULE_NOT_FOUND') {
+      return false;
+    }
+    throw error;
+  }
+};
+class LegacySqliteClient extends SqliteClient {
+  _driver() {
+    return require('sqlite3');
+  }
+}
+
+const clientMap = {
+  'better-sqlite3': 'better-sqlite3',
+  '@vscode/sqlite3': 'sqlite',
+  sqlite3: LegacySqliteClient,
+};
+
+const getSqlitePackageName = () => {
+  // NOTE: allow forcing the package to use (mostly used for testing purposes)
+  if (typeof process.env.SQLITE_PKG !== 'undefined') {
+    return process.env.SQLITE_PKG;
+  }
+
+  // NOTE: this tries to find the best sqlite module possible to use
+  // while keeping retro compatibility
+  return (
+    trySqlitePackage('better-sqlite3') ||
+    trySqlitePackage('@vscode/sqlite3') ||
+    trySqlitePackage('sqlite3')
+  );
+};
+
+const createConnection = config => {
+  const knexConfig = { ...config };
+  if (knexConfig.client === 'sqlite') {
+    const sqlitePackageName = getSqlitePackageName();
+
+    knexConfig.client = clientMap[sqlitePackageName];
+  }
+
+  const knexInstance = knex(knexConfig);
+
+  return Object.assign(knexInstance, {
+    getSchemaName() {
+      return this.client.connectionSettings.schema;
+    },
+  });
+};
+
+module.exports = createConnection;

packages/core/database/lib/dialects/mysql/schema-inspector.js

@@ -48,7 +48,7 @@ const toStrapiType = column => {
   switch (rootType) {
     case 'int': {
       if (column.column_key === 'PRI') {
-        return { type: 'increments', args: [{ primary: true }], unsigned: false };
+        return { type: 'increments', args: [{ primary: true, primaryKey: true }], unsigned: false };
       }
 
       return { type: 'integer' };

packages/core/database/lib/dialects/sqlite/schema-inspector.js

@@ -16,7 +16,7 @@ const toStrapiType = column => {
   switch (rootType) {
     case 'integer': {
       if (column.pk) {
-        return { type: 'increments', args: [{ primary: true }] };
+        return { type: 'increments', args: [{ primary: true, primaryKey: true }] };
       }
 
       return { type: 'integer' };

packages/core/database/lib/entity-manager.js

@@ -166,10 +166,12 @@ const createEntityManager = db => {
 
       const dataToInsert = processData(metadata, data, { withDefaults: true });
 
-      const [id] = await this.createQueryBuilder(uid)
+      const res = await this.createQueryBuilder(uid)
         .insert(dataToInsert)
         .execute();
 
+      const id = res[0].id || res[0];
+
       await this.attachRelations(uid, id, data);
 
       // TODO: in case there is no select or populate specified return the inserted data ?

packages/core/database/lib/index.js

@@ -1,28 +1,17 @@
 'use strict';
 
-const knex = require('knex');
-
 const { getDialect } = require('./dialects');
 const createSchemaProvider = require('./schema');
 const createMetadata = require('./metadata');
 const { createEntityManager } = require('./entity-manager');
 const { createMigrationsProvider } = require('./migrations');
 const { createLifecyclesProvider } = require('./lifecycles');
+const createConnection = require('./connection');
 const errors = require('./errors');
 
 // TODO: move back into strapi
 const { transformContentTypes } = require('./utils/content-types');
 
-const createConnection = config => {
-  const knexInstance = knex(config);
-
-  return Object.assign(knexInstance, {
-    getSchemaName() {
-      return this.client.connectionSettings.schema;
-    },
-  });
-};
-
 class Database {
   constructor(config) {
     this.metadata = createMetadata(config.models);

packages/core/database/lib/schema/schema.js

@@ -105,7 +105,7 @@ const getColumnType = attribute => {
     case 'increments': {
       return {
         type: 'increments',
-        args: [{ primary: true }],
+        args: [{ primary: true, primaryKey: true }],
         notNullable: true,
       };
     }

packages/core/database/package.json

@@ -34,7 +34,7 @@
     "date-fns": "2.22.1",
     "debug": "4.3.1",
     "fs-extra": "10.0.0",
-    "knex": "0.95.6",
+    "knex": "1.0.4",
     "lodash": "4.17.21",
     "umzug": "2.3.0"
   },

packages/generators/app/lib/create-cli-db-project.js

@@ -5,6 +5,7 @@ const { merge } = require('lodash');
 const { trackUsage } = require('./utils/usage');
 const defaultConfigs = require('./utils/db-configs');
 const clientDependencies = require('./utils/db-client-dependencies');
+const getClientName = require('./utils/db-client-name');
 const createProject = require('./create-project');
 
 module.exports = async scope => {
@@ -13,7 +14,7 @@ module.exports = async scope => {
 
   const client = scope.database.client;
   const configuration = {
-    client,
+    client: getClientName({ client }),
     connection: merge({}, defaultConfigs[client] || {}, scope.database),
     dependencies: clientDependencies({ scope, client }),
   };

packages/generators/app/lib/utils/db-client-dependencies.js

@@ -3,7 +3,8 @@
 const sqlClientModule = {
   mysql: { mysql: '2.18.1' },
   postgres: { pg: '8.6.0' },
-  sqlite: { sqlite3: '5.0.2' },
+  sqlite: { 'better-sqlite3': '^7.5.0' },
+  'sqlite-legacy': { sqlite3: '^5.0.2' },
 };
 
 /**
@@ -12,6 +13,7 @@ const sqlClientModule = {
 module.exports = ({ client }) => {
   switch (client) {
     case 'sqlite':
+    case 'sqlite-legacy':
     case 'postgres':
     case 'mysql':
       return {

packages/generators/app/lib/utils/db-client-name.js

@@ -0,0 +1,13 @@
+'use strict';
+
+/**
+ * Client
+ */
+module.exports = ({ client }) => {
+  switch (client) {
+    case 'sqlite-legacy':
+      return 'sqlite';
+    default:
+      return client;
+  }
+};

packages/generators/app/lib/utils/usage.js

@@ -54,6 +54,10 @@ function captureStderr(name, error) {
 }
 
 function trackEvent(event, body) {
+  if (process.env.NODE_ENV === 'test') {
+    return;
+  }
+
   try {
     return fetch('https://analytics.strapi.io/track', {
       method: 'POST',

test/create-test-app.js

@@ -1,5 +1,7 @@
 'use strict';
 
+process.env.NODE_ENV = 'test';
+
 const yargs = require('yargs');
 const { cleanTestApp, generateTestApp } = require('./helpers/test-app-generator');