[WIP] Upgrade Knex to latest version and swap unsupported sqlite3 fork

.github/workflows/tests.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [12, 14, 16]
+        node: [14, 16]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2-beta
@@ -30,7 +30,7 @@ jobs:
       CODECOV_TOKEN: ${{ secrets.codecov }}
     strategy:
       matrix:
-        node: [12, 14, 16]
+        node: [14, 16]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2-beta
@@ -50,7 +50,7 @@ jobs:
       CODECOV_TOKEN: ${{ secrets.codecov }}
     strategy:
       matrix:
-        node: [12, 14, 16]
+        node: [14, 16]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2-beta
@@ -70,7 +70,7 @@ jobs:
     name: '[CE] E2E (postgres, node: ${{ matrix.node }})'
     strategy:
       matrix:
-        node: [12, 14, 16]
+        node: [14, 16]
       max-parallel: 3
     services:
       postgres:
@@ -106,7 +106,7 @@ jobs:
     name: '[CE] E2E (mysql, node: ${{ matrix.node }})'
     strategy:
       matrix:
-        node: [12, 14, 16]
+        node: [14, 16]
       max-parallel: 3
     services:
       mysql:
@@ -142,7 +142,7 @@ jobs:
     name: '[CE] E2E (mysql:5 , node: ${{ matrix.node }})'
     strategy:
       matrix:
-        node: [12, 14, 16]
+        node: [14, 16]
       max-parallel: 3
     services:
       mysql:
@@ -172,13 +172,13 @@ jobs:
         with:
           dbOptions: '--dbclient=mysql --dbhost=localhost --dbport=3306 --dbname=strapi_test --dbusername=strapi --dbpassword=strapi'
 
-  e2e_ce_sqlite:
+  e2e_ce_sqlite3:
     runs-on: ubuntu-latest
     needs: [lint, unit_back, unit_front]
-    name: '[CE] E2E (sqlite, node: ${{ matrix.node }})'
+    name: '[CE] E2E (sqlite3, node: ${{ matrix.node }})'
     strategy:
       matrix:
-        node: [12, 14, 16]
+        node: [14, 16]
       max-parallel: 3
     steps:
       - uses: actions/checkout@v2
@@ -189,7 +189,26 @@ jobs:
       - uses: ./.github/actions/install-modules
       - uses: ./.github/actions/run-e2e-tests
         with:
-          dbOptions: '--dbclient=sqlite --dbfile=./tmp/data.db'
+          dbOptions: '--dbclient=sqlite3 --dbfile=./tmp/data.db'
+
+  e2e_ce_better_sqlite3:
+    runs-on: ubuntu-latest
+    needs: [lint, unit_back, unit_front]
+    name: '[CE] E2E (better_sqlite3, node: ${{ matrix.node }})'
+    strategy:
+      matrix:
+        node: [14, 16]
+      max-parallel: 3
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node }}
+          cache: yarn
+      - uses: ./.github/actions/install-modules
+      - uses: ./.github/actions/run-e2e-tests
+        with:
+          dbOptions: '--dbclient=better-sqlite3 --dbfile=./tmp/data.db'
 
   # EE
   e2e_ee_pg:
@@ -201,7 +220,7 @@ jobs:
       STRAPI_LICENSE: ${{ secrets.strapiLicense }}
     strategy:
       matrix:
-        node: [12, 14, 16]
+        node: [14, 16]
       max-parallel: 3
     services:
       postgres:
@@ -242,7 +261,7 @@ jobs:
       STRAPI_LICENSE: ${{ secrets.strapiLicense }}
     strategy:
       matrix:
-        node: [12, 14, 16]
+        node: [14, 16]
       max-parallel: 3
     services:
       mysql:
@@ -276,13 +295,36 @@ jobs:
   e2e_ee_sqlite:
     runs-on: ubuntu-latest
     needs: [lint, unit_back, unit_front]
-    name: '[EE] E2E (sqlite, node: ${{ matrix.node }})'
+    name: '[EE] E2E (sqlite3, node: ${{ matrix.node }})'
+    if: github.event.pull_request.head.repo.full_name == github.repository && !(github.actor == 'dependabot[bot]' || github.actor == 'dependabot-preview[bot]')
+    env:
+      STRAPI_LICENSE: ${{ secrets.strapiLicense }}
+    strategy:
+      matrix:
+        node: [14, 16]
+      max-parallel: 3
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node }}
+          cache: yarn
+      - uses: ./.github/actions/install-modules
+      - uses: ./.github/actions/run-e2e-tests
+        with:
+          dbOptions: '--dbclient=sqlite3 --dbfile=./tmp/data.db'
+          runEE: true
+
+  e2e_ee_better_sqlite:
+    runs-on: ubuntu-latest
+    needs: [lint, unit_back, unit_front]
+    name: '[EE] E2E (better_sqlite3, node: ${{ matrix.node }})'
     if: github.event.pull_request.head.repo.full_name == github.repository && !(github.actor == 'dependabot[bot]' || github.actor == 'dependabot-preview[bot]')
     env:
       STRAPI_LICENSE: ${{ secrets.strapiLicense }}
     strategy:
       matrix:
-        node: [12, 14, 16]
+        node: [14, 16]
       max-parallel: 3
     steps:
       - uses: actions/checkout@v2
@@ -293,5 +335,5 @@ jobs:
       - uses: ./.github/actions/install-modules
       - uses: ./.github/actions/run-e2e-tests
         with:
-          dbOptions: '--dbclient=sqlite --dbfile=./tmp/data.db'
+          dbOptions: '--dbclient=better-sqlite3 --dbfile=./tmp/data.db'
           runEE: true

CONTRIBUTING.md

@@ -125,21 +125,23 @@ The administration panel should now be available at http://localhost:4000/admin.
 The end-to-end tests require a Strapi app to be able to run. You can generate a "test app" using `yarn test:generate-app <database>`:
 
 ```bash
-$ yarn test:generate-app sqlite
+$ yarn test:generate-app sqlite3
 $ yarn test:generate-app postgres
 $ yarn test:generate-app mysql
+$ yarn test:generate-app mysql2
 ```
 
 A new app is required every time you run the end-to-end tests otherwise, the test suite will fail. A script is available to make this process easier: `node test/e2e.js`. It will delete the current test app, generate a new one and run the test suite.
 
 ### Changing the database
 
-By default the script `test/e2e,js` creates an app that uses `sqlite`. But you can run the test suites using different databases:
+By default the script `test/e2e,js` creates an app that uses `sqlite3`. But you can run the test suites using different databases:
 
 ```bash
-$ node test/e2e.js --db=sqlite
+$ node test/e2e.js --db=sqlite3
 $ node test/e2e.js --db=postgres
 $ node test/e2e.js --db=mysql
+$ node test/e2e.js --db=mysql2
 ```
 
 ### Running the tests for the Community Editon (CE)\*\*

package.json

@@ -77,6 +77,7 @@
     "@testing-library/user-event": "13.5.0",
     "axios-mock-adapter": "1.20.0",
     "babel-eslint": "10.1.0",
+    "better-sqlite3": "7.5.0",
     "chalk": "4.1.2",
     "chokidar": "3.5.3",
     "cross-env": "7.0.3",

packages/cli/create-strapi-app/create-strapi-app.js

@@ -34,7 +34,7 @@ program
   .option('--dbusername <dbusername>', 'Database username')
   .option('--dbpassword <dbpassword>', 'Database password')
   .option('--dbssl <dbssl>', 'Database SSL')
-  .option('--dbfile <dbfile>', 'Database file path for sqlite')
+  .option('--dbfile <dbfile>', 'Database file path for sqlite3')
   .option('--dbforce', 'Overwrite database content if any')
   .option('--template <templateurl>', 'Specify a Strapi template')
   .description('create a new application')

packages/cli/create-strapi-starter/create-strapi-starter.js

@@ -33,7 +33,7 @@ program
   .option('--dbusername <dbusername>', 'Database username')
   .option('--dbpassword <dbpassword>', 'Database password')
   .option('--dbssl <dbssl>', 'Database SSL')
-  .option('--dbfile <dbfile>', 'Database file path for sqlite')
+  .option('--dbfile <dbfile>', 'Database file path for sqlite3')
   .option('--dbforce', 'Overwrite database content if any')
   .description(
     'Create a fullstack monorepo application using the strapi backend template specified in the provided starter'

packages/core/admin/admin/src/components/GuidedTour/Homepage/index.js

@@ -38,9 +38,9 @@ const GuidedTourHomepage = () => {
   const activeSection = enrichedSections.find(section => !section.isDone)?.key;
 
   const handleSkip = () => {
-    setSkipped(true)
+    setSkipped(true);
     trackUsage('didSkipGuidedtour');
-  }
+  };
 
   return (
     <Box

packages/core/database/lib/dialects/index.js

@@ -5,9 +5,11 @@ const getDialectClass = client => {
     case 'postgres':
       return require('./postgresql');
     case 'mysql':
+    case 'mysql2':
       return require('./mysql');
-    case 'sqlite':
-      return require('./sqlite');
+    case 'sqlite3':
+    case 'better-sqlite3':
+      return require('./sqlite3');
     default:
       throw new Error(`Unknown dialect ${client}`);
   }

packages/core/database/lib/entity-manager.js

@@ -170,12 +170,12 @@ const createEntityManager = db => {
         .insert(dataToInsert)
         .execute();
 
-      await this.attachRelations(uid, id, data);
+      await this.attachRelations(uid, id.id, data);
 
       // TODO: in case there is no select or populate specified return the inserted data ?
-      // TODO: do not trigger the findOne lifecycles ?
+      // TODO: do not trigger the findOne lifecycles
       const result = await this.findOne(uid, {
-        where: { id },
+        where: { ...id },
         select: params.select,
         populate: params.populate,
       });

packages/core/database/lib/query/helpers/search.js

@@ -41,7 +41,8 @@ const applySearch = (knex, query, ctx) => {
 
       break;
     }
-    case 'sqlite': {
+    case 'sqlite3':
+    case 'better-sqlite3': {
       searchColumns.forEach(attr => {
         const columnName = toColumnName(meta, attr);
         return knex.orWhereRaw(`?? LIKE ? ESCAPE '\\'`, [
@@ -51,7 +52,8 @@ const applySearch = (knex, query, ctx) => {
       });
       break;
     }
-    case 'mysql': {
+    case 'mysql':
+    case 'mysql2': {
       searchColumns.forEach(attr => {
         const columnName = toColumnName(meta, attr);
         return knex.orWhereRaw(`?? LIKE ?`, [

packages/core/database/lib/schema/schema.js

@@ -73,6 +73,10 @@ const createTable = meta => {
     } else if (types.isScalar(attribute.type)) {
       const column = createColumn(attribute.columnName || key, attribute);
 
+      if (column.name === 'id') {
+        column.primary = true;
+      }
+
       if (column.unique) {
         table.indexes.push({
           type: 'unique',

packages/core/database/package.json

@@ -34,7 +34,7 @@
     "date-fns": "2.22.1",
     "debug": "4.3.1",
     "fs-extra": "10.0.0",
-    "knex": "0.95.6",
+    "knex": "1.0.4",
     "lodash": "4.17.21",
     "umzug": "2.3.0"
   },

packages/core/strapi/bin/strapi.js

@@ -93,7 +93,7 @@ program
   .option('--dbusername <dbusername>', 'Database username')
   .option('--dbpassword <dbpassword>', 'Database password')
   .option('--dbssl <dbssl>', 'Database SSL')
-  .option('--dbfile <dbfile>', 'Database file path for sqlite')
+  .option('--dbfile <dbfile>', 'Database file path for sqlite3')
   .option('--dbforce', 'Allow overwriting existing database content')
   .description('Create a new application')
   .action(require('../lib/commands/new'));

packages/core/strapi/tests/filtering.test.e2e.js

@@ -386,7 +386,7 @@ describe('Filtering API', () => {
       });
     });
 
-    // FIXME: Not working on sqlite due to https://www.sqlite.org/draft/pragma.html#pragma_case_sensitive_like
+    // FIXME: Not working on sqlite3 due to https://www.sqlite.org/draft/pragma.html#pragma_case_sensitive_like
     describe('Filter contains sensitive', () => {
       test.skip('Should return empty if the case does not match', async () => {
         const res = await rq({
@@ -421,7 +421,7 @@ describe('Filtering API', () => {
       });
     });
 
-    // FIXME: Not working on sqlite due to https://www.sqlite.org/draft/pragma.html#pragma_case_sensitive_like
+    // FIXME: Not working on sqlite3 due to https://www.sqlite.org/draft/pragma.html#pragma_case_sensitive_like
     describe('Filter not contains sensitive', () => {
       test.skip('Should return the entities if the case does not match', async () => {
         const res = await rq({

packages/generators/app/lib/create-customized-project.js

@@ -92,7 +92,7 @@ async function askDbInfosAndTest(scope) {
 async function testDatabaseConnection({ scope, configuration }) {
   const { client } = configuration;
 
-  if (client === 'sqlite') return;
+  if (client === 'sqlite3' || client === 'better-sqlite3') return;
 
   return;
 
@@ -122,8 +122,8 @@ async function askDatabaseInfos(scope) {
       type: 'list',
       name: 'client',
       message: 'Choose your default database client',
-      choices: ['sqlite', 'postgres', 'mysql'],
-      default: 'sqlite',
+      choices: ['better-sqlite3', 'sqlite3', 'postgres', 'mysql', 'mysql2'],
+      default: 'better-sqlite3',
     },
   ]);
 

packages/generators/app/lib/create-quickstart-project.js

@@ -12,8 +12,8 @@ module.exports = async function createQuickStartProject(scope) {
   console.log('Creating a quickstart project.');
   await trackUsage({ event: 'didChooseQuickstart', scope });
 
-  // get default sqlite config
-  const client = 'sqlite';
+  // get default sqlite3 config
+  const client = 'better-sqlite3';
   const configuration = {
     client,
     connection: defaultConfigs[client],

packages/generators/app/lib/generate-new.js

@@ -23,7 +23,7 @@ module.exports = async scope => {
     return createCLIDatabaseProject(scope);
   }
 
-  // if cli quickstart create project with default sqlite options
+  // if cli quickstart create project with default sqlite3 options
   if (scope.quick === true) {
     return createQuickStartProject(scope);
   }

packages/generators/app/lib/resources/templates/database-templates/better-sqlite3.template

@@ -0,0 +1,11 @@
+const path = require('path');
+
+module.exports = ({ env }) => ({
+  connection: {
+    client: 'better-sqlite3',
+    connection: {
+      filename: path.join(__dirname, '..', env('DATABASE_FILENAME', '<%= connection.filename %>')),
+    },
+    useNullAsDefault: true,
+  },
+});

packages/generators/app/lib/resources/templates/database-templates/mysql2.template

@@ -0,0 +1,13 @@
+module.exports = ({ env }) => ({
+  connection: {
+    client: 'mysql2',
+    connection: {
+      host: env('DATABASE_HOST', '<%= connection.host %>'),
+      port: env.int('DATABASE_PORT', <%= connection.port %>),
+      database: env('DATABASE_NAME', '<%= connection.database %>'),
+      user: env('DATABASE_USERNAME', '<%= connection.username %>'),
+      password: env('DATABASE_PASSWORD', '<%= connection.password %>'),
+      ssl: env.bool('DATABASE_SSL', <%= connection.ssl %>),
+    },
+  },
+});

packages/generators/app/lib/resources/templates/database-templates/sqlite.template → packages/generators/app/lib/resources/templates/database-templates/sqlite3.template

@@ -2,7 +2,7 @@ const path = require('path');
 
 module.exports = ({ env }) => ({
   connection: {
-    client: 'sqlite',
+    client: 'sqlite3',
     connection: {
       filename: path.join(__dirname, '..', env('DATABASE_FILENAME', '<%= connection.filename %>')),
     },