patch: Add Continued MySQL Support

Expanded packages/backend-common with MySQL Tests Updated packages/backend-tasks column types to conform to MySQL Updated packages/backend-tasks database tests with MySQL Updated packages/backend-tasks datetime column to work with MySQL Updated packages/create-app with a production MySQL App Config template Updated packages/e2e-test to allow for e2e testing with MySQL Updated plugins/app-backend some text columns to string Updated plugins/app-backend interval to work with MySQL Updated plugins/bazaar-backend to run db tests against MySQL Updated plugins/catalog-backend-module-incremental-ingestion to run db tests against MySQL Updated plugins/catalog-backend text columns to longtext to work with MySQL like issue suggested Updated plugins/code-coverage-backend text column to string Updated plugins/linguist-backend text column to string Updated plugins/tech-insights-backend text columns to string Updated plugins/tech-insights-backend db tests to include MySQL Added New E2E tests to run on pull requests to test against MySQL Co-authored-by: Alex Rocha <alexr1@vmware.com> Co-authored-by: David Alvarado <dalvarado@vmware.com> Co-authored-by: Shwetha Gururaj <gururajsh@vmware.com> Co-authored-by: Al <aberezovsky@vmware.com> Co-authored-by: Gerg <gcobb@vmware.com> Signed-off-by: Pete Levine A <lpete@vmware.com>
backstage · Aug 3, 2023 · b1767c4 · b1767c4
1 parent e8bebb6
commit b1767c4
Show file tree

Hide file tree

Showing 30 changed files with 308 additions and 48 deletions.
diff --git a/.changeset/yellow-trains-protect.md b/.changeset/yellow-trains-protect.md
@@ -0,0 +1,32 @@
+---
+'@backstage/plugin-catalog-backend-module-incremental-ingestion': patch
+'@backstage/plugin-code-coverage-backend': patch
+'@backstage/plugin-tech-insights-backend': patch
+'@backstage/plugin-linguist-backend': patch
+'@backstage/backend-common': patch
+'@backstage/plugin-catalog-backend': patch
+'@backstage/backend-tasks': patch
+'@backstage/plugin-bazaar-backend': patch
+'@backstage/plugin-app-backend': patch
+'e2e-test': patch
+---
+
+Expanded packages/backend-common added test to database manager specifically for MySQL for test that targets postgresql
+Updated packages/backend-tasks text columns to longtext to work
+with MySQL. MySQL requires lengths on their varchar columns. String columns in knex default to 255 characters unless otherwise provided in configuration
+Updated packages/backend-tasks database migrations, scheduler and task manager test database array to include MySQL in the tests
+Updated packages/backend-tasks task worker to add the includeOffset flag when updating the startAt column
+Updated packages/create-app with a production MySQL App Config template. This config is referenced in the new e2e Linux MySQL test
+Updated plugins/app-backend text columns to longtext to work
+with MySQL. MySQL requires lengths on their varchar columns. String columns in knex default to 255 characters unless otherwise provided in configuration
+Updated plugins/app-backend StaticAssetStore interval column to use the date_sub function to work with MySQL
+Updated plugins/bazaar-backend Database Handler tests to run db tests against MySQL
+Updated plugins/catalog-backend-module-incremental-ingestion WrapperProviders tests database array to run tests against MySQL
+db tests against MySQL
+Updated plugins/catalog-backend text columns to longtext to work
+with MySQL. MySQL requires lengths on their varchar columns. String columns in knex default to 255 characters unless otherwise provided in configuration
+Updated plugins/code-coverage-backend Migration sql text columns to string to work with MySQL. MySQL requires lengths on varchar columns. String columns default to 255 characters
+Updated plugins/linguist-backend Migration sql text columns to string to work with MySQL. MySQL requires lengths on varchar columns. String columns default to 255 characters
+Updated plugins/tech-insights-backend Migration sql text columns to string to work with MySQL. MySQL requires lengths on varchar columns. String columns default to 255 characters
+Updated plugins/tech-insights-backend FactRetriever engine database list to run against MySQL
+Updated e2e-test run command to run automated e2e tests against a MySQL service
diff --git a/.github/workflows/test-db.yml b/.github/workflows/test-db.yml
@@ -0,0 +1,26 @@
+name: "!Test Databases"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ "mysql" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: yarn
+    - run: yarn install
+    - run: yarn tsc
+    - run: yarn build:all
+    - run: yarn test
diff --git a/.github/workflows/verify_e2e-linux-mysql.yml b/.github/workflows/verify_e2e-linux-mysql.yml
@@ -0,0 +1,66 @@
+name: E2E Linux MySQL
+on:
+  # NOTE: If you change these you must update verify_e2e-linux-noop.yml as well
+  pull_request:
+    paths-ignore:
+      - '.changeset/**'
+      - 'contrib/**'
+      - 'docs/**'
+      - 'microsite/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    services:
+      mysql8:
+        image: mysql:8
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        options: >-
+          --health-cmd "mysqladmin ping -h localhost"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 3306/tcp
+
+    strategy:
+      fail-fast: true
+      matrix:
+        node-version: [16.x]
+
+    env:
+      CI: true
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    name: E2E Linux mysql ${{ matrix.node-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Configure Git
+        run: |
+          git config --global user.email noreply@backstage.io
+          git config --global user.name 'GitHub e2e user'
+      - name: use node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          registry-url: https://registry.npmjs.org/ # Needed for auth
+          cache: ''
+      - name: yarn install
+        uses: backstage/actions/yarn-install@v0.6.4
+        with:
+          cache-prefix: ${{ runner.os }}-v${{ matrix.node-version }}-${{ github.ref }}
+
+      - run: yarn tsc
+      - run: yarn backstage-cli repo build
+      - name: run E2E test
+        run: |
+          yarn e2e-test run
+        env:
+          BACKSTAGE_TEST_DISABLE_DOCKER: 1
+          BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING: mysql://root:root@localhost:${{ job.services.mysql8.ports[3306] }}/ignored
diff --git a/docs/backend-system/building-plugins-and-modules/02-testing.md b/docs/backend-system/building-plugins-and-modules/02-testing.md
@@ -118,7 +118,7 @@ describe('MyDatabaseClass', () => {
   // "physical" databases to test against is much costlier than creating the
   // "logical" databases within them that the individual tests use.
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
 
   // Just an example of how to conveniently bundle up the setup code

diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json
@@ -92,6 +92,7 @@
     "minimatch": "^5.0.0",
     "minimist": "^1.2.5",
     "morgan": "^1.10.0",
+    "mysql2": "^2.2.5",
     "node-fetch": "^2.6.7",
     "node-forge": "^1.3.1",
     "pg": "^8.3.0",

diff --git a/packages/backend-common/src/database/DatabaseManager.test.ts b/packages/backend-common/src/database/DatabaseManager.test.ts
@@ -425,6 +425,33 @@ describe('DatabaseManager', () => {
       );
     });
 
+    it('generates a database name override when prefix is not explicitly set for mysql', async () => {
+      const testManager = DatabaseManager.fromConfig(
+        new ConfigReader({
+          backend: {
+            database: {
+              client: 'mysql',
+              connection: {
+                host: 'localhost',
+                user: 'foo',
+                password: 'bar',
+                database: 'foodb',
+              },
+            },
+          },
+        }),
+      );
+
+      await testManager.forPlugin('testplugin').getClient();
+      const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
+      const [_baseConfig, overrides] = mockCalls[0];
+
+      expect(overrides).toHaveProperty(
+        'connection.database',
+        expect.stringContaining('backstage_plugin_'),
+      );
+    });
+
     it('uses values from plugin connection string if top level client should be used', async () => {
       const pluginId = 'stringoverride';
       await manager.forPlugin(pluginId).getClient();

diff --git a/packages/backend-tasks/migrations/20210928160613_init.js b/packages/backend-tasks/migrations/20210928160613_init.js
@@ -26,7 +26,7 @@ exports.up = async function up(knex) {
   await knex.schema.createTable('backstage_backend_tasks__tasks', table => {
     table.comment('Tasks used for scheduling work on multiple workers');
     table
-      .text('id')
+      .string('id')
       .primary()
       .notNullable()
       .comment('The unique ID of this particular task');

diff --git a/packages/backend-tasks/src/migrations.test.ts b/packages/backend-tasks/src/migrations.test.ts
@@ -43,7 +43,7 @@ jest.setTimeout(60_000);
 
 describe('migrations', () => {
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'MYSQL_8', 'SQLITE_3'],
   });
 
   it.each(databases.eachSupportedId())(

diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
@@ -36,7 +36,7 @@ jest.setTimeout(60_000);
 
 describe('PluginTaskManagerImpl', () => {
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
 
   beforeAll(async () => {

diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.test.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.test.ts
@@ -42,6 +42,7 @@ describe('PluginTaskSchedulerJanitor', () => {
       'POSTGRES_13',
       'POSTGRES_9',
       'SQLITE_3',
+      'MYSQL_8',
     ],
   });
 

diff --git a/packages/backend-tasks/src/tasks/TaskScheduler.test.ts b/packages/backend-tasks/src/tasks/TaskScheduler.test.ts
@@ -25,7 +25,7 @@ jest.setTimeout(60_000);
 describe('TaskScheduler', () => {
   const logger = getVoidLogger();
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
 
   async function createDatabase(

diff --git a/packages/backend-tasks/src/tasks/TaskWorker.test.ts b/packages/backend-tasks/src/tasks/TaskWorker.test.ts
@@ -28,7 +28,7 @@ jest.setTimeout(60_000);
 describe('TaskWorker', () => {
   const logger = getVoidLogger();
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
 
   beforeEach(() => {

diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts
@@ -175,11 +175,15 @@ export class TaskWorker {
       const time = new CronTime(settings.cadence)
         .sendAt()
         .minus({ seconds: 1 }) // immediately, if "* * * * * *"
-        .toUTC()
-        .toISO();
-      startAt = this.knex.client.config.client.includes('sqlite3')
-        ? this.knex.raw('datetime(?)', [time])
-        : this.knex.raw(`?`, [time]);
+        .toUTC();
+
+      if (this.knex.client.config.client.includes('sqlite3')) {
+        startAt = this.knex.raw('datetime(?)', [time.toISO()]);
+      } else if (this.knex.client.config.client.includes('mysql')) {
+        startAt = this.knex.raw(`?`, [time.toSQL({ includeOffset: false })]);
+      } else {
+        startAt = this.knex.raw(`?`, [time.toISO()]);
+      }
     } else {
       startAt = this.knex.fn.now();
     }
@@ -279,11 +283,16 @@ export class TaskWorker {
 
     let nextRun: Knex.Raw;
     if (isCron) {
-      const time = new CronTime(settings.cadence).sendAt().toUTC().toISO();
+      const time = new CronTime(settings.cadence).sendAt().toUTC();
       this.logger.debug(`task: ${this.taskId} will next occur around ${time}`);
-      nextRun = this.knex.client.config.client.includes('sqlite3')
-        ? this.knex.raw('datetime(?)', [time])
-        : this.knex.raw(`?`, [time]);
+
+      if (this.knex.client.config.client.includes('sqlite3')) {
+        nextRun = this.knex.raw('datetime(?)', [time.toISO()]);
+      } else if (this.knex.client.config.client.includes('mysql')) {
+        nextRun = this.knex.raw(`?`, [time.toSQL({ includeOffset: false })]);
+      } else {
+        nextRun = this.knex.raw(`?`, [time.toISO()]);
+      }
     } else {
       const dt = Duration.fromISO(settings.cadence).as('seconds');
       this.logger.debug(

diff --git a/packages/backend/package.json b/packages/backend/package.json
@@ -87,6 +87,7 @@
     "express-prom-bundle": "^6.3.6",
     "express-promise-router": "^4.1.0",
     "luxon": "^3.0.0",
+    "mysql2": "^2.2.5",
     "pg": "^8.3.0",
     "pg-connection-string": "^2.3.0",
     "prom-client": "^14.0.1",

diff --git a/packages/create-app/templates/default-app/app-config.production.mysql.yaml b/packages/create-app/templates/default-app/app-config.production.mysql.yaml
@@ -0,0 +1,29 @@
+app:
+  # Should be the same as backend.baseUrl when using the `app-backend` plugin.
+  baseUrl: http://localhost:7007
+
+backend:
+  # Note that the baseUrl should be the URL that the browser and other clients
+  # should use when communicating with the backend, i.e. it needs to be
+  # reachable not just from within the backend host, but from all of your
+  # callers. When its value is "http://localhost:7007", it's strictly private
+  # and can't be reached by others.
+  baseUrl: http://localhost:7007
+  # The listener can also be expressed as a single <host>:<port> string. In this case we bind to
+  # all interfaces, the most permissive setting. The right value depends on your specific deployment.
+  listen: ':7007'
+
+  # config options including ssl: https://github.com/mysqljs/mysql#connection-options
+  database:
+    client: mysql2
+    connection:
+      host: ${MYSQL_HOST}
+      port: ${MYSQL_PORT}
+      user: ${MYSQL_USER}
+      password: ${MYSQL_PASSWORD}
+
+catalog:
+  # Overrides the default list locations from app-config.yaml as these contain example data.
+  # See https://backstage.io/docs/features/software-catalog/#adding-components-to-the-catalog for more details
+  # on how to get entities into the catalog.
+  locations: []
diff --git a/packages/e2e-test/package.json b/packages/e2e-test/package.json
@@ -35,6 +35,7 @@
     "cross-fetch": "^3.1.5",
     "fs-extra": "10.1.0",
     "handlebars": "^4.7.3",
+    "mysql2": "^2.2.5",
     "pgtools": "^0.3.0",
     "puppeteer": "^17.0.0",
     "tree-kill": "^1.2.2"