From de084ebeb6215667141ded1eff50e187cf5b6fc2 Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 10:43:04 +0200 Subject: [PATCH 01/40] ci: only test db2 --- .github/workflows/ci.yml | 313 +-------------------------------------- 1 file changed, 3 insertions(+), 310 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfc61745af5e..bdf0120e8f99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [14, 16] + node-version: [16] name: Upload install and build artifact (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest steps: @@ -34,105 +34,14 @@ jobs: name: install-build-artifact-node-${{ matrix.node-version }} path: install-build-node-${{ matrix.node-version }}.tar retention-days: 1 - lint: - name: Lint code - runs-on: ubuntu-latest - needs: install-and-build - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 16.x - cache: yarn - - uses: actions/download-artifact@v3 - with: - name: install-build-artifact-node-16 - - name: Extract artifact - run: tar -xf install-build-node-16.tar - - run: yarn lint-no-fix - unit-test: - strategy: - fail-fast: false - matrix: - node-version: [14, 16] - name: Unit test all dialects (Node ${{ matrix.node-version }}) - runs-on: ubuntu-latest - needs: lint - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: yarn - - uses: actions/download-artifact@v3 - with: - name: install-build-artifact-node-${{ matrix.node-version }} - - name: Extract artifact - run: tar -xf install-build-node-${{ matrix.node-version }}.tar - - name: Unit tests (mariadb) - run: yarn test-unit-mariadb - - name: Unit tests (mysql) - run: yarn test-unit-mysql - - name: Unit tests (postgres) - run: yarn test-unit-postgres - - name: Unit tests (postgres-native) - run: yarn test-unit-postgres-native - - name: Unit tests (mssql) - run: yarn test-unit-mssql - - name: Unit tests (db2) - run: yarn test-unit-db2 - - name: Unit tests (ibmi) - run: yarn test-unit-ibmi - - name: Unit tests (snowflake) - run: yarn test-unit-snowflake - docs: - name: Generate TypeDoc - runs-on: ubuntu-latest - needs: lint - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 16.x - cache: yarn - - uses: actions/download-artifact@v3 - with: - name: install-build-artifact-node-16 - - name: Extract artifact - run: tar -xf install-build-node-16.tar - - run: yarn docs - test-typings: - strategy: - fail-fast: false - matrix: - ts-version: ["4.4", "4.5", "4.6"] - name: TS Typings (${{ matrix.ts-version }}) - runs-on: ubuntu-latest - needs: lint - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 16.x - cache: yarn - - uses: actions/download-artifact@v3 - with: - name: install-build-artifact-node-16 - - name: Extract artifact - run: tar -xf install-build-node-16.tar - # This step uses npm instead of yarn to minimize the time needed. See #14171 - - name: Install TypeScript - run: npm install --no-save --no-audit typescript@~${{ matrix.ts-version }} - - name: Typing Tests - run: yarn test-typings test-db2: strategy: fail-fast: false matrix: - node-version: [14, 16] + node-version: [16] name: DB2 (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest - needs: [ unit-test, test-typings ] + needs: [ install-and-build ] env: DIALECT: db2 SEQ_DB: testdb @@ -155,219 +64,3 @@ jobs: run: yarn start-db2 - name: Integration Tests run: yarn test-integration - test-sqlite: - strategy: - fail-fast: false - matrix: - node-version: [14, 16] - name: SQLite (Node ${{ matrix.node-version }}) - runs-on: ubuntu-latest - needs: [ unit-test, test-typings ] - env: - DIALECT: sqlite - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: yarn - - uses: actions/download-artifact@v3 - with: - name: install-build-artifact-node-${{ matrix.node-version }} - - name: Extract artifact - run: tar -xf install-build-node-${{ matrix.node-version }}.tar - - name: Integration Tests - run: yarn test-integration - test-postgres: - strategy: - fail-fast: false - matrix: - node-version: [14, 16] - postgres-version: [9.5, 10] - minify-aliases: [true, false] - native: [true, false] - name: Postgres ${{ matrix.postgres-version }}${{ matrix.native && ' (native)' || '' }} (Node ${{ matrix.node-version }})${{ matrix.minify-aliases && ' (minified aliases)' || '' }} - runs-on: ubuntu-latest - needs: [ unit-test, test-typings ] - services: - postgres: - image: sushantdhiman/postgres:${{ matrix.postgres-version }} - env: - POSTGRES_USER: sequelize_test - POSTGRES_DB: sequelize_test - POSTGRES_PASSWORD: sequelize_test - ports: - - 5432:5432 - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - env: - SEQ_PORT: 5432 - DIALECT: ${{ matrix.native && 'postgres-native' || 'postgres' }} - SEQ_PG_MINIFY_ALIASES: ${{ matrix.minify-aliases && '1' || '' }} - steps: - - run: PGPASSWORD=sequelize_test psql -h localhost -p 5432 -U sequelize_test sequelize_test -c '\l' - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: yarn - - uses: actions/download-artifact@v3 - with: - name: install-build-artifact-node-${{ matrix.node-version }} - - name: Extract artifact - run: tar -xf install-build-node-${{ matrix.node-version }}.tar - - name: Install pg-native (Node 14) - run: yarn add pg-native --ignore-engines - if: matrix.native && matrix.node-version == 14 - # This step uses npm instead of yarn to minimize the time needed. See #14171 - - name: Install pg-native (Node 16) - run: npm install --no-save --no-audit pg-native - if: matrix.native && matrix.node-version == 16 - - name: Integration Tests - run: yarn test-integration - test-mysql-mariadb: - strategy: - fail-fast: false - matrix: - include: - - name: MySQL 5.7 - image: mysql:5.7.37 - dialect: mysql - node-version: 14 - - name: MySQL 5.7 - image: mysql:5.7.37 - dialect: mysql - node-version: 16 - - name: MySQL 8.0 - image: mysql:8.0.28 - dialect: mysql - node-version: 14 - - name: MySQL 8.0 - image: mysql:8.0.28 - dialect: mysql - node-version: 16 - - name: MariaDB 10.3 - image: mariadb:10.3.34 - dialect: mariadb - node-version: 14 - - name: MariaDB 10.3 - image: mariadb:10.3.34 - dialect: mariadb - node-version: 16 - - name: MariaDB 10.5 - image: mariadb:10.5 - dialect: mariadb - node-version: 14 - - name: MariaDB 10.5 - image: mariadb:10.5 - dialect: mariadb - node-version: 16 - name: ${{ matrix.name }} (Node ${{ matrix.node-version }}) - runs-on: ubuntu-latest - needs: [ unit-test, test-typings ] - services: - mysql: - image: ${{ matrix.image }} - env: - MYSQL_DATABASE: sequelize_test - MYSQL_USER: sequelize_test - MYSQL_PASSWORD: sequelize_test - MYSQL_ROOT_PASSWORD: sequelize_test - ports: - - 3306:3306 - options: --health-cmd="mysqladmin -usequelize_test -psequelize_test status" --health-interval 10s --health-timeout 5s --health-retries 5 --tmpfs /var/lib/mysql:rw - env: - SEQ_PORT: 3306 - DIALECT: ${{ matrix.dialect }} - steps: - - run: mysql --host 127.0.0.1 --port 3306 -uroot -psequelize_test -e "GRANT ALL ON *.* TO 'sequelize_test'@'%' with grant option; FLUSH PRIVILEGES;" - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: yarn - - uses: actions/download-artifact@v3 - with: - name: install-build-artifact-node-${{ matrix.node-version }} - - name: Extract artifact - run: tar -xf install-build-node-${{ matrix.node-version }}.tar - - name: Integration Tests - run: yarn test-integration - test-mssql: - strategy: - fail-fast: false - matrix: - node-version: [14, 16] - mssql-version: [2017, 2019] - name: MSSQL ${{ matrix.mssql-version }} (Node ${{ matrix.node-version }}) - runs-on: ubuntu-latest - needs: [ unit-test, test-typings ] - services: - mssql: - image: mcr.microsoft.com/mssql/server:${{ matrix.mssql-version }}-latest - env: - ACCEPT_EULA: Y - SA_PASSWORD: Password12! - ports: - - 1433:1433 - options: >- - --health-cmd="/opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P \"Password12!\" -l 30 -Q \"SELECT 1\"" - --health-start-period 10s - --health-interval 10s - --health-timeout 5s - --health-retries 10 - env: - DIALECT: mssql - SEQ_USER: SA - SEQ_PW: Password12! - SEQ_PORT: 1433 - steps: - - run: /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "Password12!" -Q "CREATE DATABASE sequelize_test; ALTER DATABASE sequelize_test SET READ_COMMITTED_SNAPSHOT ON;" - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: yarn - - uses: actions/download-artifact@v3 - with: - name: install-build-artifact-node-${{ matrix.node-version }} - - name: Extract artifact - run: tar -xf install-build-node-${{ matrix.node-version }}.tar - - name: Integration Tests - run: yarn test-integration - release: - name: Release - runs-on: ubuntu-latest - needs: - [ - docs, - test-sqlite, - test-postgres, - test-mysql-mariadb, - test-mssql, - ] - if: github.event_name == 'push' && (github.ref == 'refs/heads/v6' || github.ref == 'refs/heads/v7') - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 16.x - cache: yarn - - uses: actions/download-artifact@v3 - with: - name: install-build-artifact-node-16 - - name: Extract artifact - run: tar -xf install-build-node-16.tar - - run: yarn semantic-release - - id: sequelize - uses: sdepold/github-action-get-latest-release@master - with: - repository: sequelize/sequelize - - name: Notify channels - run: | - curl -XPOST -u "sdepold:${{ secrets.GH_TOKEN }}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/sequelize/sequelize/dispatches --data '{"event_type":"Release notifier","client_payload":{"release-id": ${{ steps.sequelize.outputs.id }}}}' - - name: Notify docs repo - run: | - curl -XPOST -u "sdepold:${{ secrets.GH_TOKEN }}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/sequelize/website/dispatches --data '{"event_type":"Build website"}' From f4c4e3d5cfb8f99064ff0a86519b044d363610bd Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 10:58:28 +0200 Subject: [PATCH 02/40] ci: update db2 image --- dev/db2/11.5/start.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/db2/11.5/start.sh b/dev/db2/11.5/start.sh index 110907b49c7a..edae45933688 100644 --- a/dev/db2/11.5/start.sh +++ b/dev/db2/11.5/start.sh @@ -3,22 +3,22 @@ export DIALECT=db2 mkdir -p Docker if [ ! "$(sudo docker ps -q -f name=db2server)" ]; then - if [ "$(sudo docker ps -aq -f status=exited -f name=db2server)" ]; + if [ "$(sudo docker ps -aq -f status=exited -f name=db2server)" ]; then # cleanup sudo docker rm -f db2server sudo rm -rf /Docker fi - sudo docker run -h db2server --name db2server --restart=always --detach --privileged=true -p 50000:50000 --env-file .env_list -v /Docker:/database ibmcom/db2-amd64:11.5.6.0a + sudo docker run -h db2server --name db2server --restart=always --detach --privileged=true -p 50000:50000 --env-file .env_list -v /Docker:/database ibmcom/db2-amd64:11.5.7.0 count=1 while true do if (sudo docker logs db2server | grep 'Setup has completed') - then + then sudo docker exec db2server bash -c "su db2inst1 & disown" break fi - if ($count -gt 30); then + if [ $count -gt 30 ]; then echo "Error: Db2 docker setup has not completed in 10 minutes." break fi From a3a860a1dc2785794c0f95ddc82303ddcb1cb07d Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 11:09:15 +0200 Subject: [PATCH 03/40] only some tests --- test/integration/model/sync.test.js | 2 +- test/integration/query-interface/changeColumn.test.js | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integration/model/sync.test.js b/test/integration/model/sync.test.js index aa0c6ada1dea..fb1a0050ca19 100644 --- a/test/integration/model/sync.test.js +++ b/test/integration/model/sync.test.js @@ -193,7 +193,7 @@ describe(Support.getTestDialectTeaser('Model'), () => { describe('indexes', () => { describe('with alter:true', () => { - it('should not duplicate named indexes after multiple sync calls', async function () { + it.only('should not duplicate named indexes after multiple sync calls', async function () { const User = this.sequelize.define('testSync', { email: { type: DataTypes.STRING, diff --git a/test/integration/query-interface/changeColumn.test.js b/test/integration/query-interface/changeColumn.test.js index 6a857eea295d..8b8e8f2f7c38 100644 --- a/test/integration/query-interface/changeColumn.test.js +++ b/test/integration/query-interface/changeColumn.test.js @@ -8,7 +8,7 @@ const { DataTypes } = require('@sequelize/core'); const dialect = Support.getTestDialect(); -describe(Support.getTestDialectTeaser('QueryInterface'), () => { +describe.only(Support.getTestDialectTeaser('QueryInterface'), () => { beforeEach(function () { this.sequelize.options.quoteIdenifiers = true; this.queryInterface = this.sequelize.getQueryInterface(); @@ -25,6 +25,7 @@ describe(Support.getTestDialectTeaser('QueryInterface'), () => { await this.queryInterface.createTable({ tableName: 'users', schema: 'archive', + logging: true, }, { id: { type: DataTypes.INTEGER, @@ -37,6 +38,7 @@ describe(Support.getTestDialectTeaser('QueryInterface'), () => { await this.queryInterface.changeColumn({ tableName: 'users', schema: 'archive', + logging: true, }, 'currency', { type: DataTypes.FLOAT, }); From 5dd7fc7a94533f7d183d54a51500dcde48cda2ef Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 13:55:23 +0200 Subject: [PATCH 04/40] ci: try randomizing db2 db name --- .github/workflows/ci.yml | 4 ++-- dev/db2/11.5/.env_list | 3 +-- dev/db2/11.5/start.sh | 4 +++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bdf0120e8f99..692260613095 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,13 +38,13 @@ jobs: strategy: fail-fast: false matrix: - node-version: [16] + node-version: [14, 16] name: DB2 (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest needs: [ install-and-build ] env: DIALECT: db2 - SEQ_DB: testdb + SEQ_DB: testdb-node-${{ matrix.node-version }}-sha-${{ github.sha }} SEQ_USER: db2inst1 SEQ_PW: password SEQ_TEST_CLEANUP_TIMEOUT: 1200000 diff --git a/dev/db2/11.5/.env_list b/dev/db2/11.5/.env_list index 2cf451f91a7e..2e70ed2ccb87 100644 --- a/dev/db2/11.5/.env_list +++ b/dev/db2/11.5/.env_list @@ -1,7 +1,6 @@ LICENSE=accept DB2INSTANCE=db2inst1 DB2INST1_PASSWORD=password -DBNAME=testdb BLU=false ENABLE_ORACLE_COMPATIBILITY=false UPDATEAVAIL=NO @@ -12,4 +11,4 @@ PERSISTENT_HOME=false HADR_ENABLED=false ETCD_ENDPOINT= ETCD_USERNAME= -ETCD_PASSWORD= \ No newline at end of file +ETCD_PASSWORD= diff --git a/dev/db2/11.5/start.sh b/dev/db2/11.5/start.sh index edae45933688..c9b009f562ed 100644 --- a/dev/db2/11.5/start.sh +++ b/dev/db2/11.5/start.sh @@ -1,5 +1,7 @@ cd dev/db2/11.5 + export DIALECT=db2 +export SEQ_DB="${SEQ_DB:-testdb}" mkdir -p Docker if [ ! "$(sudo docker ps -q -f name=db2server)" ]; then @@ -9,7 +11,7 @@ if [ ! "$(sudo docker ps -q -f name=db2server)" ]; then sudo docker rm -f db2server sudo rm -rf /Docker fi - sudo docker run -h db2server --name db2server --restart=always --detach --privileged=true -p 50000:50000 --env-file .env_list -v /Docker:/database ibmcom/db2-amd64:11.5.7.0 + sudo docker run -h db2server --name db2server --restart=always --detach --privileged=true -p 50000:50000 --env "DBNAME=$SEQ_DB" --env-file .env_list -v /Docker:/database ibmcom/db2-amd64:11.5.7.0 count=1 while true do From b5a0a71288e043080ee01567a91e6a2a40def0cd Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 13:59:08 +0200 Subject: [PATCH 05/40] ci: prepare node 14 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 692260613095..d80e309eab56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [16] + node-version: [14, 16] name: Upload install and build artifact (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest steps: From 863f39e455dad8b0e5b028301077542755eabcd8 Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 14:09:42 +0200 Subject: [PATCH 06/40] ci: print db2 logs --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d80e309eab56..6e5dbb597477 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,5 +62,7 @@ jobs: run: tar -xf install-build-node-${{ matrix.node-version }}.tar - name: Install Local DB2 Copy run: yarn start-db2 + - name: Print DB2 start logs + run: sudo docker logs db2server - name: Integration Tests run: yarn test-integration From b818776d7f2e87e90f33b8cd713421b896150b52 Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 14:21:12 +0200 Subject: [PATCH 07/40] fix: uppercase db name --- dev/db2/11.5/start.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/db2/11.5/start.sh b/dev/db2/11.5/start.sh index c9b009f562ed..4c4e0fa59202 100644 --- a/dev/db2/11.5/start.sh +++ b/dev/db2/11.5/start.sh @@ -2,6 +2,8 @@ cd dev/db2/11.5 export DIALECT=db2 export SEQ_DB="${SEQ_DB:-testdb}" +# db2 db names must be uppercase +export SEQ_DB=$(echo "$SEQ_DB" | awk '{print toupper($0)}') mkdir -p Docker if [ ! "$(sudo docker ps -q -f name=db2server)" ]; then From 8a3d21a7f644b70a0fd0389a316b0226d21ea417 Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 14:30:54 +0200 Subject: [PATCH 08/40] ci: use _ instead of - in db name --- .github/workflows/ci.yml | 2 +- dev/db2/11.5/start.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e5dbb597477..257940b14988 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: needs: [ install-and-build ] env: DIALECT: db2 - SEQ_DB: testdb-node-${{ matrix.node-version }}-sha-${{ github.sha }} + SEQ_DB: testdb_node_${{ matrix.node-version }}_sha_${{ github.sha }} SEQ_USER: db2inst1 SEQ_PW: password SEQ_TEST_CLEANUP_TIMEOUT: 1200000 diff --git a/dev/db2/11.5/start.sh b/dev/db2/11.5/start.sh index 4c4e0fa59202..7fcb88522fd8 100644 --- a/dev/db2/11.5/start.sh +++ b/dev/db2/11.5/start.sh @@ -1,9 +1,9 @@ cd dev/db2/11.5 export DIALECT=db2 -export SEQ_DB="${SEQ_DB:-testdb}" +SEQ_DB="${SEQ_DB:-testdb}" # db2 db names must be uppercase -export SEQ_DB=$(echo "$SEQ_DB" | awk '{print toupper($0)}') +SEQ_DB=$(echo "$SEQ_DB" | awk '{print toupper($0)}') mkdir -p Docker if [ ! "$(sudo docker ps -q -f name=db2server)" ]; then From 43aee15d8d152b7aa8cf191d59c49945720896ca Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 14:36:48 +0200 Subject: [PATCH 09/40] ci: apparently db2 does not accept _ either --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 257940b14988..b1dd8abe3386 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: needs: [ install-and-build ] env: DIALECT: db2 - SEQ_DB: testdb_node_${{ matrix.node-version }}_sha_${{ github.sha }} + SEQ_DB: testdbnode${{ matrix.node-version }}sha${{ github.sha }} SEQ_USER: db2inst1 SEQ_PW: password SEQ_TEST_CLEANUP_TIMEOUT: 1200000 From 4c9c5463f9400e001e19f6204d576b7087f5ee29 Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 14:52:37 +0200 Subject: [PATCH 10/40] ci: generate smaller DB name --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1dd8abe3386..8e59400ebe42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,11 @@ jobs: name: install-build-artifact-node-${{ matrix.node-version }} - name: Extract artifact run: tar -xf install-build-node-${{ matrix.node-version }}.tar + - name: Generate DB name + run: | + echo "SEQ_DB=n${{ matrix.node-version }}s$(git rev-parse --short HEAD)" >> $GITHUB_ENV + - name: Print DB name + run: echo "DB Name is $SEQ_DB" - name: Install Local DB2 Copy run: yarn start-db2 - name: Print DB2 start logs From fdd6fa1bf0420a6963a78556c815dc5b9f402541 Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 15:10:15 +0200 Subject: [PATCH 11/40] ci: generate shorter db name --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e59400ebe42..29e4bf074539 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,12 @@ jobs: run: tar -xf install-build-node-${{ matrix.node-version }}.tar - name: Generate DB name run: | - echo "SEQ_DB=n${{ matrix.node-version }}s$(git rev-parse --short HEAD)" >> $GITHUB_ENV + NODE_VERSION="${{ matrix.node-version }}" + SHA=$(git rev-parse --short HEAD) + + # db2 database names cannot be longer than 8 chars. + # start with n prefix + last character of node version + 6 first characters of github sha + echo "SEQ_DB=n${NODE_VERSION: -1}${SHA:0:6}" >> $GITHUB_ENV - name: Print DB name run: echo "DB Name is $SEQ_DB" - name: Install Local DB2 Copy From 0ea6f299dd2d27a7401fb33ba38f555e79c249a3 Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 15:15:27 +0200 Subject: [PATCH 12/40] test: reenable tests --- test/integration/model/sync.test.js | 2 +- test/integration/query-interface/changeColumn.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/model/sync.test.js b/test/integration/model/sync.test.js index fb1a0050ca19..aa0c6ada1dea 100644 --- a/test/integration/model/sync.test.js +++ b/test/integration/model/sync.test.js @@ -193,7 +193,7 @@ describe(Support.getTestDialectTeaser('Model'), () => { describe('indexes', () => { describe('with alter:true', () => { - it.only('should not duplicate named indexes after multiple sync calls', async function () { + it('should not duplicate named indexes after multiple sync calls', async function () { const User = this.sequelize.define('testSync', { email: { type: DataTypes.STRING, diff --git a/test/integration/query-interface/changeColumn.test.js b/test/integration/query-interface/changeColumn.test.js index 8b8e8f2f7c38..161f141be614 100644 --- a/test/integration/query-interface/changeColumn.test.js +++ b/test/integration/query-interface/changeColumn.test.js @@ -8,7 +8,7 @@ const { DataTypes } = require('@sequelize/core'); const dialect = Support.getTestDialect(); -describe.only(Support.getTestDialectTeaser('QueryInterface'), () => { +describe(Support.getTestDialectTeaser('QueryInterface'), () => { beforeEach(function () { this.sequelize.options.quoteIdenifiers = true; this.queryInterface = this.sequelize.getQueryInterface(); From 77b3011a351fa27742e5601b98f98549edd1f951 Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 15:50:13 +0200 Subject: [PATCH 13/40] test: small tweaks to see if changeColumn still errors --- .../query-interface/changeColumn.test.js | 6 +----- test/support.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/test/integration/query-interface/changeColumn.test.js b/test/integration/query-interface/changeColumn.test.js index 161f141be614..021f8207d5b8 100644 --- a/test/integration/query-interface/changeColumn.test.js +++ b/test/integration/query-interface/changeColumn.test.js @@ -9,12 +9,9 @@ const { DataTypes } = require('@sequelize/core'); const dialect = Support.getTestDialect(); describe(Support.getTestDialectTeaser('QueryInterface'), () => { - beforeEach(function () { + beforeEach(async function () { this.sequelize.options.quoteIdenifiers = true; this.queryInterface = this.sequelize.getQueryInterface(); - }); - - afterEach(async function () { await Support.dropTestSchemas(this.sequelize); }); @@ -38,7 +35,6 @@ describe(Support.getTestDialectTeaser('QueryInterface'), () => { await this.queryInterface.changeColumn({ tableName: 'users', schema: 'archive', - logging: true, }, 'currency', { type: DataTypes.FLOAT, }); diff --git a/test/support.ts b/test/support.ts index b35281647088..73ed8b26967a 100644 --- a/test/support.ts +++ b/test/support.ts @@ -1,3 +1,4 @@ +import assert from 'assert'; import fs from 'fs'; import path from 'path'; import { inspect, isDeepStrictEqual } from 'util'; @@ -188,7 +189,17 @@ export async function dropTestSchemas(sequelize: Sequelize) { } } - await Promise.all(schemasPromise.map(async p => p.catch((error: unknown) => error))); + const results = await Promise.allSettled(schemasPromise); + const errors = results.filter(result => result.status === 'rejected') + .map(result => { + assert(result.status === 'rejected'); + + return result.reason; + }); + + if (errors.length > 0) { + throw new AggregateError(errors, 'At least one schema deletion operation errored'); + } } export function getSupportedDialects() { From 53fe653315e0b88bb7403455c97fd73933afcbee Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 15:59:50 +0200 Subject: [PATCH 14/40] test: log all queries --- test/support.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/support.ts b/test/support.ts index 73ed8b26967a..bad32953f48b 100644 --- a/test/support.ts +++ b/test/support.ts @@ -123,7 +123,7 @@ export function createSequelizeInstance(options: Options = {}) { const sequelizeOptions = defaults(options, { host: options.host || config.host, - logging: process.env.SEQ_LOG ? console.debug : false, + logging: console.log, dialect: options.dialect, port: options.port || process.env.SEQ_PORT || config.port, pool: config.pool, From 050fb8eebad81ea7daf1b8c4b93fda56817caa14 Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 16:34:27 +0200 Subject: [PATCH 15/40] feat: let's see what happens if we don't mute errors --- src/dialects/db2/query.js | 108 ++++++++++++++++++-------------------- test/support.ts | 13 +---- 2 files changed, 51 insertions(+), 70 deletions(-) diff --git a/src/dialects/db2/query.js b/src/dialects/db2/query.js index 4697ca06db5b..c818f998c227 100644 --- a/src/dialects/db2/query.js +++ b/src/dialects/db2/query.js @@ -109,14 +109,6 @@ export class Db2Query extends AbstractQuery { this.sequelize.log(`Executed (${this.connection.uuid || 'default'}): ${newSql} ${parameters ? JSON.stringify(parameters) : ''}`, Date.now() - queryBegin, this.options); } - if (err && err.message) { - err = this.filterSQLError(err, this.sql, connection); - if (err === null) { - stmt.closeSync(); - resolve(this.formatResults([], 0)); - } - } - if (err) { err.sql = sql; stmt.closeSync(); @@ -201,56 +193,56 @@ export class Db2Query extends AbstractQuery { return [sql, bindParam]; } - filterSQLError(err, sql, connection) { - if (err.message.search('SQL0204N') !== -1 && _.startsWith(sql, 'DROP ')) { - err = null; // Ignore table not found error for drop table. - } else if (err.message.search('SQL0443N') !== -1) { - if (this.isDropSchemaQuery()) { - // Delete ERRORSCHEMA.ERRORTABLE if it exist. - connection.querySync('DROP TABLE ERRORSCHEMA.ERRORTABLE;'); - // Retry deleting the schema - connection.querySync(this.sql); - } - - err = null; // Ignore drop schema error. - } else if (err.message.search('SQL0601N') !== -1) { - const match = err.message.match(/SQL0601N {2}The name of the object to be created is identical to the existing name "(.*)" of type "(.*)"./); - if (match && match.length > 1 && match[2] === 'TABLE') { - let table; - const mtarray = match[1].split('.'); - if (mtarray[1]) { - table = `"${mtarray[0]}"."${mtarray[1]}"`; - } else { - table = `"${mtarray[0]}"`; - } - - if (connection.dropTable !== false) { - connection.querySync(`DROP TABLE ${table}`); - err = connection.querySync(sql); - } else { - err = null; - } - } else { - err = null; // Ignore create schema error. - } - } else if (err.message.search('SQL0911N') !== -1) { - if (err.message.search('Reason code "2"') !== -1) { - err = null; // Ignore deadlock error due to program logic. - } - } else if (err.message.search('SQL0605W') !== -1) { - err = null; // Ignore warning. - } else if (err.message.search('SQL0668N') !== -1 - && _.startsWith(sql, 'ALTER TABLE ')) { - connection.querySync(`CALL SYSPROC.ADMIN_CMD('REORG TABLE ${sql.slice(12).split(' ')[0]}')`); - err = connection.querySync(sql); - } - - if (err && err.length === 0) { - err = null; - } - - return err; - } + // filterSQLError(err, sql, connection) { + // if (err.message.search('SQL0204N') !== -1 && _.startsWith(sql, 'DROP ')) { + // err = null; // Ignore table not found error for drop table. + // } else if (err.message.search('SQL0443N') !== -1) { + // if (this.isDropSchemaQuery()) { + // // Delete ERRORSCHEMA.ERRORTABLE if it exist. + // connection.querySync('DROP TABLE ERRORSCHEMA.ERRORTABLE;'); + // // Retry deleting the schema + // connection.querySync(this.sql); + // } + // + // err = null; // Ignore drop schema error. + // } else if (err.message.search('SQL0601N') !== -1) { + // const match = err.message.match(/SQL0601N {2}The name of the object to be created is identical to the existing name "(.*)" of type "(.*)"./); + // if (match && match.length > 1 && match[2] === 'TABLE') { + // let table; + // const mtarray = match[1].split('.'); + // if (mtarray[1]) { + // table = `"${mtarray[0]}"."${mtarray[1]}"`; + // } else { + // table = `"${mtarray[0]}"`; + // } + // + // if (connection.dropTable !== false) { + // connection.querySync(`DROP TABLE ${table}`); + // err = connection.querySync(sql); + // } else { + // err = null; + // } + // } else { + // err = null; // Ignore create schema error. + // } + // } else if (err.message.search('SQL0911N') !== -1) { + // if (err.message.search('Reason code "2"') !== -1) { + // err = null; // Ignore deadlock error due to program logic. + // } + // } else if (err.message.search('SQL0605W') !== -1) { + // err = null; // Ignore warning. + // } else if (err.message.search('SQL0668N') !== -1 + // && _.startsWith(sql, 'ALTER TABLE ')) { + // connection.querySync(`CALL SYSPROC.ADMIN_CMD('REORG TABLE ${sql.slice(12).split(' ')[0]}')`); + // err = connection.querySync(sql); + // } + // + // if (err && err.length === 0) { + // err = null; + // } + // + // return err; + // } /** * High level function that handles the results of a query execution. diff --git a/test/support.ts b/test/support.ts index bad32953f48b..5b5d916dda43 100644 --- a/test/support.ts +++ b/test/support.ts @@ -1,4 +1,3 @@ -import assert from 'assert'; import fs from 'fs'; import path from 'path'; import { inspect, isDeepStrictEqual } from 'util'; @@ -189,17 +188,7 @@ export async function dropTestSchemas(sequelize: Sequelize) { } } - const results = await Promise.allSettled(schemasPromise); - const errors = results.filter(result => result.status === 'rejected') - .map(result => { - assert(result.status === 'rejected'); - - return result.reason; - }); - - if (errors.length > 0) { - throw new AggregateError(errors, 'At least one schema deletion operation errored'); - } + await Promise.all(schemasPromise); } export function getSupportedDialects() { From 8037ebcbec3443340c5928ba3c4222922e0e39dc Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 16:48:55 +0200 Subject: [PATCH 16/40] fix: use 'if exists' in db2 drop table --- src/dialects/db2/query-generator.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/dialects/db2/query-generator.js b/src/dialects/db2/query-generator.js index 94cbcded62e9..0b5ce031cbd7 100644 --- a/src/dialects/db2/query-generator.js +++ b/src/dialects/db2/query-generator.js @@ -180,15 +180,6 @@ export class Db2QueryGenerator extends AbstractQueryGenerator { return 'SELECT TABNAME AS "tableName", TRIM(TABSCHEMA) AS "tableSchema" FROM SYSCAT.TABLES WHERE TABSCHEMA = USER AND TYPE = \'T\' ORDER BY TABSCHEMA, TABNAME'; } - dropTableQuery(tableName) { - const query = 'DROP TABLE <%= table %>'; - const values = { - table: this.quoteTable(tableName), - }; - - return `${_.template(query, this._templateSettings)(values).trim()};`; - } - addColumnQuery(table, key, dataType) { dataType.field = key; From c1643539b96dace45dfc8d34a2f687ecc87f2c7f Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 26 Apr 2022 17:05:25 +0200 Subject: [PATCH 17/40] fix: more attempts idk --- src/dialects/db2/query-interface.js | 7 ++ src/dialects/db2/query.js | 114 ++++++++++++++++------------ test/integration/support.js | 18 ++++- 3 files changed, 88 insertions(+), 51 deletions(-) diff --git a/src/dialects/db2/query-interface.js b/src/dialects/db2/query-interface.js index d69e7eee7a05..6923870810cb 100644 --- a/src/dialects/db2/query-interface.js +++ b/src/dialects/db2/query-interface.js @@ -76,6 +76,13 @@ export class Db2QueryInterface extends QueryInterface { return [result, undefined]; } + async dropSchema(schema, options) { + // error from dropSchema will be stored in this table. + await this.dropTable({ tableName: 'ERRORTABLE', schema: 'ERRORSCHEMA' }); + + return super.dropSchema(schema, options); + } + async createTable(tableName, attributes, options, model) { let sql = ''; diff --git a/src/dialects/db2/query.js b/src/dialects/db2/query.js index c818f998c227..298543d3d316 100644 --- a/src/dialects/db2/query.js +++ b/src/dialects/db2/query.js @@ -109,6 +109,14 @@ export class Db2Query extends AbstractQuery { this.sequelize.log(`Executed (${this.connection.uuid || 'default'}): ${newSql} ${parameters ? JSON.stringify(parameters) : ''}`, Date.now() - queryBegin, this.options); } + if (err && err.message) { + err = this.filterSQLError(err, this.sql, connection); + if (err === null) { + stmt.closeSync(); + resolve(this.formatResults([], 0)); + } + } + if (err) { err.sql = sql; stmt.closeSync(); @@ -193,56 +201,62 @@ export class Db2Query extends AbstractQuery { return [sql, bindParam]; } - // filterSQLError(err, sql, connection) { - // if (err.message.search('SQL0204N') !== -1 && _.startsWith(sql, 'DROP ')) { - // err = null; // Ignore table not found error for drop table. - // } else if (err.message.search('SQL0443N') !== -1) { - // if (this.isDropSchemaQuery()) { - // // Delete ERRORSCHEMA.ERRORTABLE if it exist. - // connection.querySync('DROP TABLE ERRORSCHEMA.ERRORTABLE;'); - // // Retry deleting the schema - // connection.querySync(this.sql); - // } - // - // err = null; // Ignore drop schema error. - // } else if (err.message.search('SQL0601N') !== -1) { - // const match = err.message.match(/SQL0601N {2}The name of the object to be created is identical to the existing name "(.*)" of type "(.*)"./); - // if (match && match.length > 1 && match[2] === 'TABLE') { - // let table; - // const mtarray = match[1].split('.'); - // if (mtarray[1]) { - // table = `"${mtarray[0]}"."${mtarray[1]}"`; - // } else { - // table = `"${mtarray[0]}"`; - // } - // - // if (connection.dropTable !== false) { - // connection.querySync(`DROP TABLE ${table}`); - // err = connection.querySync(sql); - // } else { - // err = null; - // } - // } else { - // err = null; // Ignore create schema error. - // } - // } else if (err.message.search('SQL0911N') !== -1) { - // if (err.message.search('Reason code "2"') !== -1) { - // err = null; // Ignore deadlock error due to program logic. - // } - // } else if (err.message.search('SQL0605W') !== -1) { - // err = null; // Ignore warning. - // } else if (err.message.search('SQL0668N') !== -1 - // && _.startsWith(sql, 'ALTER TABLE ')) { - // connection.querySync(`CALL SYSPROC.ADMIN_CMD('REORG TABLE ${sql.slice(12).split(' ')[0]}')`); - // err = connection.querySync(sql); - // } - // - // if (err && err.length === 0) { - // err = null; - // } - // - // return err; - // } + filterSQLError(err, sql, connection) { + // This error is safe to ignore: + // [IBM][CLI Driver][DB2/LINUXX8664] SQL0605W The index was not created because an index "x" with a matching definition already exists. SQLSTATE=01550 + if (err.message.search('SQL0605W') !== -1) { + return null; + } + + // if (err.message.search('SQL0204N') !== -1 && _.startsWith(sql, 'DROP ')) { + // err = null; // Ignore table not found error for drop table. + // } else if (err.message.search('SQL0443N') !== -1) { + // if (this.isDropSchemaQuery()) { + // // Delete ERRORSCHEMA.ERRORTABLE if it exist. + // connection.querySync('DROP TABLE ERRORSCHEMA.ERRORTABLE;'); + // // Retry deleting the schema + // connection.querySync(this.sql); + // } + // + // err = null; // Ignore drop schema error. + // } else if (err.message.search('SQL0601N') !== -1) { + // const match = err.message.match(/SQL0601N {2}The name of the object to be created is identical to the existing name "(.*)" of type "(.*)"./); + // if (match && match.length > 1 && match[2] === 'TABLE') { + // let table; + // const mtarray = match[1].split('.'); + // if (mtarray[1]) { + // table = `"${mtarray[0]}"."${mtarray[1]}"`; + // } else { + // table = `"${mtarray[0]}"`; + // } + // + // if (connection.dropTable !== false) { + // connection.querySync(`DROP TABLE ${table}`); + // err = connection.querySync(sql); + // } else { + // err = null; + // } + // } else { + // err = null; // Ignore create schema error. + // } + // } else if (err.message.search('SQL0911N') !== -1) { + // if (err.message.search('Reason code "2"') !== -1) { + // err = null; // Ignore deadlock error due to program logic. + // } + // } else if (err.message.search('SQL0605W') !== -1) { + // err = null; // Ignore warning. + // } else if (err.message.search('SQL0668N') !== -1 + // && _.startsWith(sql, 'ALTER TABLE ')) { + // connection.querySync(`CALL SYSPROC.ADMIN_CMD('REORG TABLE ${sql.slice(12).split(' ')[0]}')`); + // err = connection.querySync(sql); + // } + // + // if (err && err.length === 0) { + // err = null; + // } + // + return err; + } /** * High level function that handles the results of a query execution. diff --git a/test/integration/support.js b/test/integration/support.js index 969e7de39c5f..d0008683c6b9 100644 --- a/test/integration/support.js +++ b/test/integration/support.js @@ -6,12 +6,28 @@ const { setTimeout, clearTimeout } = global; const pTimeout = require('p-timeout'); const Support = require('../support'); +const { getTestDialect } = require('../support'); const CLEANUP_TIMEOUT = Number.parseInt(process.env.SEQ_TEST_CLEANUP_TIMEOUT, 10) || 10_000; let runningQueries = new Set(); -before(function () { +before(async function () { + if (getTestDialect() === 'db2') { + // needed by dropSchema function + await this.sequelize.query(` + CREATE TABLESPACE SYSTOOLSPACE IN IBMCATGROUP + MANAGED BY AUTOMATIC STORAGE USING STOGROUP IBMSTOGROUP + EXTENTSIZE 4; + `); + + await this.sequelize.query(` + CREATE USER TEMPORARY TABLESPACE SYSTOOLSTMPSPACE IN IBMCATGROUP + MANAGED BY AUTOMATIC STORAGE USING STOGROUP IBMSTOGROUP + EXTENTSIZE 4 + `); + } + this.sequelize.addHook('beforeQuery', (options, query) => { runningQueries.add(query); }); From 26f0dc374ca0934a791a0de3aeea1d9baecf0625 Mon Sep 17 00:00:00 2001 From: ephys Date: Sun, 12 Jun 2022 11:31:48 +0200 Subject: [PATCH 18/40] ci: test commit From 134d74efbc964d04a8c990d5280193d75e6146e1 Mon Sep 17 00:00:00 2001 From: ephys Date: Sun, 12 Jun 2022 11:35:12 +0200 Subject: [PATCH 19/40] ci: revert changes --- .github/workflows/ci.yml | 337 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 320 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29e4bf074539..01ae08cfa600 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,6 @@ name: CI on: push: branches: - - v6 - v7 pull_request: @@ -11,12 +10,19 @@ env: SEQ_USER: sequelize_test SEQ_PW: sequelize_test +# This configuration cancels previous runs if a new run is started on the same PR. Only one run at a time per PR. +# This does not affect pushes to the v7 branch itself, only PRs. +# from https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-a-fallback-value +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: install-and-build: strategy: fail-fast: false matrix: - node-version: [14, 16] + node-version: [14, 18] name: Upload install and build artifact (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest steps: @@ -34,17 +40,110 @@ jobs: name: install-build-artifact-node-${{ matrix.node-version }} path: install-build-node-${{ matrix.node-version }}.tar retention-days: 1 + lint: + name: Lint code + runs-on: ubuntu-latest + needs: install-and-build + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: yarn + - uses: actions/download-artifact@v3 + with: + name: install-build-artifact-node-18 + - name: Extract artifact + run: tar -xf install-build-node-18.tar + - run: yarn lint-no-fix + unit-test: + strategy: + fail-fast: false + matrix: + node-version: [14, 18] + name: Unit test all dialects (Node ${{ matrix.node-version }}) + runs-on: ubuntu-latest + needs: lint + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: yarn + - uses: actions/download-artifact@v3 + with: + name: install-build-artifact-node-${{ matrix.node-version }} + - name: Extract artifact + run: tar -xf install-build-node-${{ matrix.node-version }}.tar + - name: Unit tests (mariadb) + run: yarn test-unit-mariadb + - name: Unit tests (mysql) + run: yarn test-unit-mysql + - name: Unit tests (postgres) + run: yarn test-unit-postgres + - name: Unit tests (postgres-native) + run: yarn test-unit-postgres-native + - name: Unit tests (sqlite) + run: yarn test-unit-sqlite + - name: Unit tests (mssql) + run: yarn test-unit-mssql + - name: Unit tests (db2) + run: yarn test-unit-db2 + - name: Unit tests (ibmi) + run: yarn test-unit-ibmi + - name: Unit tests (snowflake) + run: yarn test-unit-snowflake + docs: + name: Generate TypeDoc + runs-on: ubuntu-latest + needs: lint + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: yarn + - uses: actions/download-artifact@v3 + with: + name: install-build-artifact-node-18 + - name: Extract artifact + run: tar -xf install-build-node-18.tar + - run: yarn docs + test-typings: + strategy: + fail-fast: false + matrix: + ts-version: ["4.4", "4.5", "4.6", "4.7"] + name: TS Typings (${{ matrix.ts-version }}) + runs-on: ubuntu-latest + needs: lint + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: yarn + - uses: actions/download-artifact@v3 + with: + name: install-build-artifact-node-18 + - name: Extract artifact + run: tar -xf install-build-node-18.tar + # This step uses npm instead of yarn to minimize the time needed. See #14171 + - name: Install TypeScript + run: npm install --no-save --no-audit typescript@~${{ matrix.ts-version }} + - name: Typing Tests + run: yarn test-typings test-db2: strategy: fail-fast: false matrix: - node-version: [14, 16] + node-version: [14, 18] name: DB2 (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest - needs: [ install-and-build ] + needs: [ unit-test, test-typings ] env: DIALECT: db2 - SEQ_DB: testdbnode${{ matrix.node-version }}sha${{ github.sha }} + SEQ_DB: testdb SEQ_USER: db2inst1 SEQ_PW: password SEQ_TEST_CLEANUP_TIMEOUT: 1200000 @@ -60,19 +159,223 @@ jobs: name: install-build-artifact-node-${{ matrix.node-version }} - name: Extract artifact run: tar -xf install-build-node-${{ matrix.node-version }}.tar - - name: Generate DB name - run: | - NODE_VERSION="${{ matrix.node-version }}" - SHA=$(git rev-parse --short HEAD) - - # db2 database names cannot be longer than 8 chars. - # start with n prefix + last character of node version + 6 first characters of github sha - echo "SEQ_DB=n${NODE_VERSION: -1}${SHA:0:6}" >> $GITHUB_ENV - - name: Print DB name - run: echo "DB Name is $SEQ_DB" - name: Install Local DB2 Copy run: yarn start-db2 - - name: Print DB2 start logs - run: sudo docker logs db2server - name: Integration Tests run: yarn test-integration + test-sqlite: + strategy: + fail-fast: false + matrix: + node-version: [14, 18] + name: SQLite (Node ${{ matrix.node-version }}) + runs-on: ubuntu-latest + needs: [ unit-test, test-typings ] + env: + DIALECT: sqlite + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: yarn + - uses: actions/download-artifact@v3 + with: + name: install-build-artifact-node-${{ matrix.node-version }} + - name: Extract artifact + run: tar -xf install-build-node-${{ matrix.node-version }}.tar + - name: Integration Tests + run: yarn test-integration + test-postgres: + strategy: + fail-fast: false + matrix: + node-version: [14, 18] + postgres-version: [9.5, 10] + minify-aliases: [true, false] + native: [true, false] + name: Postgres ${{ matrix.postgres-version }}${{ matrix.native && ' (native)' || '' }} (Node ${{ matrix.node-version }})${{ matrix.minify-aliases && ' (minified aliases)' || '' }} + runs-on: ubuntu-latest + needs: [ unit-test, test-typings ] + services: + postgres: + image: sushantdhiman/postgres:${{ matrix.postgres-version }} + env: + POSTGRES_USER: sequelize_test + POSTGRES_DB: sequelize_test + POSTGRES_PASSWORD: sequelize_test + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + env: + SEQ_PORT: 5432 + DIALECT: ${{ matrix.native && 'postgres-native' || 'postgres' }} + SEQ_PG_MINIFY_ALIASES: ${{ matrix.minify-aliases && '1' || '' }} + steps: + - run: PGPASSWORD=sequelize_test psql -h localhost -p 5432 -U sequelize_test sequelize_test -c '\l' + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: yarn + - uses: actions/download-artifact@v3 + with: + name: install-build-artifact-node-${{ matrix.node-version }} + - name: Extract artifact + run: tar -xf install-build-node-${{ matrix.node-version }}.tar + - name: Install pg-native (Node 14) + run: yarn add pg-native --ignore-engines + if: matrix.native && matrix.node-version == 14 + # This step uses npm instead of yarn to minimize the time needed. See #14171 + - name: Install pg-native (Node 18) + run: npm install --no-save --no-audit pg-native + if: matrix.native && matrix.node-version == 18 + - name: Integration Tests + run: yarn test-integration + test-mysql-mariadb: + strategy: + fail-fast: false + matrix: + include: + - name: MySQL 5.7 + image: mysql:5.7.37 + dialect: mysql + node-version: 14 + - name: MySQL 5.7 + image: mysql:5.7.37 + dialect: mysql + node-version: 18 + - name: MySQL 8.0 + image: mysql:8.0.28 + dialect: mysql + node-version: 14 + - name: MySQL 8.0 + image: mysql:8.0.28 + dialect: mysql + node-version: 18 + - name: MariaDB 10.3 + image: mariadb:10.3.34 + dialect: mariadb + node-version: 14 + - name: MariaDB 10.3 + image: mariadb:10.3.34 + dialect: mariadb + node-version: 18 + - name: MariaDB 10.5 + image: mariadb:10.5 + dialect: mariadb + node-version: 14 + - name: MariaDB 10.5 + image: mariadb:10.5 + dialect: mariadb + node-version: 18 + name: ${{ matrix.name }} (Node ${{ matrix.node-version }}) + runs-on: ubuntu-latest + needs: [ unit-test, test-typings ] + services: + mysql: + image: ${{ matrix.image }} + env: + MYSQL_DATABASE: sequelize_test + MYSQL_USER: sequelize_test + MYSQL_PASSWORD: sequelize_test + MYSQL_ROOT_PASSWORD: sequelize_test + ports: + - 3306:3306 + options: --health-cmd="mysqladmin -usequelize_test -psequelize_test status" --health-interval 10s --health-timeout 5s --health-retries 5 --tmpfs /var/lib/mysql:rw + env: + SEQ_PORT: 3306 + DIALECT: ${{ matrix.dialect }} + steps: + - run: mysql --host 127.0.0.1 --port 3306 -uroot -psequelize_test -e "GRANT ALL ON *.* TO 'sequelize_test'@'%' with grant option; FLUSH PRIVILEGES;" + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: yarn + - uses: actions/download-artifact@v3 + with: + name: install-build-artifact-node-${{ matrix.node-version }} + - name: Extract artifact + run: tar -xf install-build-node-${{ matrix.node-version }}.tar + - name: Integration Tests + run: yarn test-integration + test-mssql: + strategy: + fail-fast: false + matrix: + node-version: [14, 18] + mssql-version: [2017, 2022] + name: MSSQL ${{ matrix.mssql-version }} (Node ${{ matrix.node-version }}) + runs-on: ubuntu-latest + needs: [ unit-test, test-typings ] + services: + mssql: + image: mcr.microsoft.com/mssql/server:${{ matrix.mssql-version }}-latest + env: + ACCEPT_EULA: Y + SA_PASSWORD: Password12! + ports: + - 1433:1433 + options: >- + --health-cmd="/opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P \"Password12!\" -l 30 -Q \"SELECT 1\"" + --health-start-period 10s + --health-interval 10s + --health-timeout 5s + --health-retries 10 + env: + DIALECT: mssql + SEQ_USER: SA + SEQ_PW: Password12! + SEQ_PORT: 1433 + steps: + - run: /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "Password12!" -Q "CREATE DATABASE sequelize_test; ALTER DATABASE sequelize_test SET READ_COMMITTED_SNAPSHOT ON;" + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: yarn + - uses: actions/download-artifact@v3 + with: + name: install-build-artifact-node-${{ matrix.node-version }} + - name: Extract artifact + run: tar -xf install-build-node-${{ matrix.node-version }}.tar + - name: Integration Tests + run: yarn test-integration + release: + name: Release + runs-on: ubuntu-latest + needs: + [ + docs, + test-sqlite, + test-postgres, + test-mysql-mariadb, + test-mssql, + ] + if: github.event_name == 'push' && (github.ref == 'refs/heads/v6' || github.ref == 'refs/heads/v7') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: yarn + - uses: actions/download-artifact@v3 + with: + name: install-build-artifact-node-18 + - name: Extract artifact + run: tar -xf install-build-node-18.tar + - run: yarn semantic-release + - id: sequelize + uses: sdepold/github-action-get-latest-release@master + with: + repository: sequelize/sequelize + - name: Notify channels + run: | + curl -XPOST -u "sdepold:${{ secrets.GH_TOKEN }}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/sequelize/sequelize/dispatches --data '{"event_type":"Release notifier","client_payload":{"release-id": ${{ steps.sequelize.outputs.id }}}}' + - name: Notify docs repo + run: | + curl -XPOST -u "sdepold:${{ secrets.GH_TOKEN }}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/sequelize/website/dispatches --data '{"event_type":"Build website"}' From 566aaba5d56c1e53e3aece73fbcf2db51aeab41f Mon Sep 17 00:00:00 2001 From: ephys Date: Sun, 12 Jun 2022 11:43:57 +0200 Subject: [PATCH 20/40] test: remove console.log --- test/integration/query-interface/changeColumn.test.js | 1 - test/support.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/test/integration/query-interface/changeColumn.test.js b/test/integration/query-interface/changeColumn.test.js index 021f8207d5b8..c3e2d1774874 100644 --- a/test/integration/query-interface/changeColumn.test.js +++ b/test/integration/query-interface/changeColumn.test.js @@ -22,7 +22,6 @@ describe(Support.getTestDialectTeaser('QueryInterface'), () => { await this.queryInterface.createTable({ tableName: 'users', schema: 'archive', - logging: true, }, { id: { type: DataTypes.INTEGER, diff --git a/test/support.ts b/test/support.ts index 5b5d916dda43..464f4dd4d3c5 100644 --- a/test/support.ts +++ b/test/support.ts @@ -122,7 +122,7 @@ export function createSequelizeInstance(options: Options = {}) { const sequelizeOptions = defaults(options, { host: options.host || config.host, - logging: console.log, + logging: process.env.SEQ_LOG ? console.debug : false, dialect: options.dialect, port: options.port || process.env.SEQ_PORT || config.port, pool: config.pool, From 6474da06306203a282341a45e63cca37db870f7e Mon Sep 17 00:00:00 2001 From: ephys Date: Sun, 12 Jun 2022 11:46:22 +0200 Subject: [PATCH 21/40] test: fix db2 test --- test/unit/dialects/db2/query-generator.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/dialects/db2/query-generator.test.js b/test/unit/dialects/db2/query-generator.test.js index 04903dd0cf91..ff22f9bf4d17 100644 --- a/test/unit/dialects/db2/query-generator.test.js +++ b/test/unit/dialects/db2/query-generator.test.js @@ -166,7 +166,7 @@ if (dialect === 'db2') { dropTableQuery: [ { arguments: ['myTable'], - expectation: 'DROP TABLE "myTable";', + expectation: 'DROP TABLE IF EXISTS "myTable";', }, ], From fbbfd9dad6c8d3d1ef1cb914954a5d1f5b277d5c Mon Sep 17 00:00:00 2001 From: ephys Date: Sun, 12 Jun 2022 12:19:26 +0200 Subject: [PATCH 22/40] fix(db2): use if-not-exists during create table --- src/dialects/db2/query-generator.js | 2 +- .../unit/dialects/db2/query-generator.test.js | 22 +++++++++---------- test/unit/sql/create-table.test.js | 6 ++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/dialects/db2/query-generator.js b/src/dialects/db2/query-generator.js index 418a89d8a6dc..60a461dea47a 100644 --- a/src/dialects/db2/query-generator.js +++ b/src/dialects/db2/query-generator.js @@ -58,7 +58,7 @@ export class Db2QueryGenerator extends AbstractQueryGenerator { } createTableQuery(tableName, attributes, options) { - const query = 'CREATE TABLE <%= table %> (<%= attributes %>)'; + const query = 'CREATE TABLE IF NOT EXISTS <%= table %> (<%= attributes %>)'; const primaryKeys = []; const foreignKeys = {}; const attrStr = []; diff --git a/test/unit/dialects/db2/query-generator.test.js b/test/unit/dialects/db2/query-generator.test.js index ff22f9bf4d17..ef0e4bd40486 100644 --- a/test/unit/dialects/db2/query-generator.test.js +++ b/test/unit/dialects/db2/query-generator.test.js @@ -119,47 +119,47 @@ if (dialect === 'db2') { createTableQuery: [ { arguments: ['myTable', { title: 'VARCHAR(255)', name: 'VARCHAR(255)' }], - expectation: 'CREATE TABLE "myTable" ("title" VARCHAR(255), "name" VARCHAR(255));', + expectation: 'CREATE TABLE IF NOT EXISTS "myTable" ("title" VARCHAR(255), "name" VARCHAR(255));', }, { arguments: ['myTable', { data: 'BLOB' }], - expectation: 'CREATE TABLE "myTable" ("data" BLOB);', + expectation: 'CREATE TABLE IF NOT EXISTS "myTable" ("data" BLOB);', }, { arguments: ['myTable', { data: 'BLOB(16M)' }], - expectation: 'CREATE TABLE "myTable" ("data" BLOB(16M));', + expectation: 'CREATE TABLE IF NOT EXISTS "myTable" ("data" BLOB(16M));', }, { arguments: ['myTable', { title: 'VARCHAR(255)', name: 'VARCHAR(255)' }, { engine: 'MyISAM' }], - expectation: 'CREATE TABLE "myTable" ("title" VARCHAR(255), "name" VARCHAR(255));', + expectation: 'CREATE TABLE IF NOT EXISTS "myTable" ("title" VARCHAR(255), "name" VARCHAR(255));', }, { arguments: ['myTable', { title: 'VARCHAR(255)', name: 'VARCHAR(255)' }, { charset: 'utf8', collate: 'utf8_unicode_ci' }], - expectation: 'CREATE TABLE "myTable" ("title" VARCHAR(255), "name" VARCHAR(255));', + expectation: 'CREATE TABLE IF NOT EXISTS "myTable" ("title" VARCHAR(255), "name" VARCHAR(255));', }, { arguments: ['myTable', { title: 'VARCHAR(255)', name: 'VARCHAR(255)' }, { charset: 'latin1' }], - expectation: 'CREATE TABLE "myTable" ("title" VARCHAR(255), "name" VARCHAR(255));', + expectation: 'CREATE TABLE IF NOT EXISTS "myTable" ("title" VARCHAR(255), "name" VARCHAR(255));', }, { arguments: ['myTable', { title: 'ENUM("A", "B", "C")', name: 'VARCHAR(255)' }, { charset: 'latin1' }], - expectation: 'CREATE TABLE "myTable" ("title" ENUM("A", "B", "C"), "name" VARCHAR(255));', + expectation: 'CREATE TABLE IF NOT EXISTS "myTable" ("title" ENUM("A", "B", "C"), "name" VARCHAR(255));', }, { arguments: ['myTable', { title: 'VARCHAR(255)', name: 'VARCHAR(255)' }, { rowFormat: 'default' }], - expectation: 'CREATE TABLE "myTable" ("title" VARCHAR(255), "name" VARCHAR(255));', + expectation: 'CREATE TABLE IF NOT EXISTS "myTable" ("title" VARCHAR(255), "name" VARCHAR(255));', }, { arguments: ['myTable', { title: 'VARCHAR(255)', name: 'VARCHAR(255)', id: 'INTEGER PRIMARY KEY' }], - expectation: 'CREATE TABLE "myTable" ("title" VARCHAR(255), "name" VARCHAR(255), "id" INTEGER , PRIMARY KEY ("id"));', + expectation: 'CREATE TABLE IF NOT EXISTS "myTable" ("title" VARCHAR(255), "name" VARCHAR(255), "id" INTEGER , PRIMARY KEY ("id"));', }, { arguments: ['myTable', { title: 'VARCHAR(255)', name: 'VARCHAR(255)', otherId: 'INTEGER REFERENCES otherTable (id) ON DELETE CASCADE ON UPDATE NO ACTION' }], - expectation: 'CREATE TABLE "myTable" ("title" VARCHAR(255), "name" VARCHAR(255), "otherId" INTEGER, FOREIGN KEY ("otherId") REFERENCES otherTable (id) ON DELETE CASCADE ON UPDATE NO ACTION);', + expectation: 'CREATE TABLE IF NOT EXISTS "myTable" ("title" VARCHAR(255), "name" VARCHAR(255), "otherId" INTEGER, FOREIGN KEY ("otherId") REFERENCES otherTable (id) ON DELETE CASCADE ON UPDATE NO ACTION);', }, { arguments: ['myTable', { title: 'VARCHAR(255)', name: 'VARCHAR(255)' }, { uniqueKeys: [{ fields: ['title', 'name'], customIndex: true }] }], - expectation: 'CREATE TABLE "myTable" ("title" VARCHAR(255) NOT NULL, "name" VARCHAR(255) NOT NULL, CONSTRAINT "uniq_myTable_title_name" UNIQUE ("title", "name"));', + expectation: 'CREATE TABLE IF NOT EXISTS "myTable" ("title" VARCHAR(255) NOT NULL, "name" VARCHAR(255) NOT NULL, CONSTRAINT "uniq_myTable_title_name" UNIQUE ("title", "name"));', }, ], diff --git a/test/unit/sql/create-table.test.js b/test/unit/sql/create-table.test.js index 2a7e7235bb46..5bf641580f25 100644 --- a/test/unit/sql/create-table.test.js +++ b/test/unit/sql/create-table.test.js @@ -25,7 +25,7 @@ describe(Support.getTestDialectTeaser('SQL'), () => { it('references enum in the right schema #3171', () => { expectsql(sql.createTableQuery(FooUser.getTableName(), sql.attributesToSQL(FooUser.rawAttributes), {}), { sqlite: 'CREATE TABLE IF NOT EXISTS `foo.users` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `mood` TEXT);', - db2: 'CREATE TABLE "foo"."users" ("id" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY(START WITH 1, INCREMENT BY 1) , "mood" VARCHAR(255) CHECK ("mood" IN(\'happy\', \'sad\')), PRIMARY KEY ("id"));', + db2: 'CREATE TABLE IF NOT EXISTS "foo"."users" ("id" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY(START WITH 1, INCREMENT BY 1) , "mood" VARCHAR(255) CHECK ("mood" IN(\'happy\', \'sad\')), PRIMARY KEY ("id"));', postgres: 'CREATE TABLE IF NOT EXISTS "foo"."users" ("id" SERIAL , "mood" "foo"."enum_users_mood", PRIMARY KEY ("id"));', mariadb: 'CREATE TABLE IF NOT EXISTS `foo`.`users` (`id` INTEGER NOT NULL auto_increment , `mood` ENUM(\'happy\', \'sad\'), PRIMARY KEY (`id`)) ENGINE=InnoDB;', mysql: 'CREATE TABLE IF NOT EXISTS `foo.users` (`id` INTEGER NOT NULL auto_increment , `mood` ENUM(\'happy\', \'sad\'), PRIMARY KEY (`id`)) ENGINE=InnoDB;', @@ -60,7 +60,7 @@ describe(Support.getTestDialectTeaser('SQL'), () => { it('references right schema when adding foreign key #9029', () => { expectsql(sql.createTableQuery(BarProject.getTableName(), sql.attributesToSQL(BarProject.rawAttributes), {}), { sqlite: 'CREATE TABLE IF NOT EXISTS `bar.projects` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `user_id` INTEGER REFERENCES `bar.users` (`id`) ON DELETE NO ACTION ON UPDATE CASCADE);', - db2: 'CREATE TABLE "bar"."projects" ("id" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY(START WITH 1, INCREMENT BY 1) , "user_id" INTEGER, PRIMARY KEY ("id"), FOREIGN KEY ("user_id") REFERENCES "bar"."users" ("id") ON DELETE NO ACTION);', + db2: 'CREATE TABLE IF NOT EXISTS "bar"."projects" ("id" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY(START WITH 1, INCREMENT BY 1) , "user_id" INTEGER, PRIMARY KEY ("id"), FOREIGN KEY ("user_id") REFERENCES "bar"."users" ("id") ON DELETE NO ACTION);', postgres: 'CREATE TABLE IF NOT EXISTS "bar"."projects" ("id" SERIAL , "user_id" INTEGER REFERENCES "bar"."users" ("id") ON DELETE NO ACTION ON UPDATE CASCADE, PRIMARY KEY ("id"));', mariadb: 'CREATE TABLE IF NOT EXISTS `bar`.`projects` (`id` INTEGER NOT NULL auto_increment , `user_id` INTEGER, PRIMARY KEY (`id`), FOREIGN KEY (`user_id`) REFERENCES `bar`.`users` (`id`) ON DELETE NO ACTION ON UPDATE CASCADE) ENGINE=InnoDB;', mysql: 'CREATE TABLE IF NOT EXISTS `bar.projects` (`id` INTEGER NOT NULL auto_increment , `user_id` INTEGER, PRIMARY KEY (`id`), FOREIGN KEY (`user_id`) REFERENCES `bar.users` (`id`) ON DELETE NO ACTION ON UPDATE CASCADE) ENGINE=InnoDB;', @@ -94,7 +94,7 @@ describe(Support.getTestDialectTeaser('SQL'), () => { expectsql(sql.createTableQuery(Image.getTableName(), sql.attributesToSQL(Image.rawAttributes), {}), { sqlite: 'CREATE TABLE IF NOT EXISTS `images` (`id` INTEGER PRIMARY KEY AUTOINCREMENT REFERENCES `files` (`id`));', postgres: 'CREATE TABLE IF NOT EXISTS "images" ("id" SERIAL REFERENCES "files" ("id"), PRIMARY KEY ("id"));', - db2: 'CREATE TABLE "images" ("id" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY(START WITH 1, INCREMENT BY 1) , PRIMARY KEY ("id"), FOREIGN KEY ("id") REFERENCES "files" ("id"));', + db2: 'CREATE TABLE IF NOT EXISTS "images" ("id" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY(START WITH 1, INCREMENT BY 1) , PRIMARY KEY ("id"), FOREIGN KEY ("id") REFERENCES "files" ("id"));', mariadb: 'CREATE TABLE IF NOT EXISTS `images` (`id` INTEGER auto_increment , PRIMARY KEY (`id`), FOREIGN KEY (`id`) REFERENCES `files` (`id`)) ENGINE=InnoDB;', mysql: 'CREATE TABLE IF NOT EXISTS `images` (`id` INTEGER auto_increment , PRIMARY KEY (`id`), FOREIGN KEY (`id`) REFERENCES `files` (`id`)) ENGINE=InnoDB;', mssql: 'IF OBJECT_ID(\'[images]\', \'U\') IS NULL CREATE TABLE [images] ([id] INTEGER IDENTITY(1,1) , PRIMARY KEY ([id]), FOREIGN KEY ([id]) REFERENCES [files] ([id]));', From 9feb422fd9fbdff86d1f760e360ff1c84a6958cc Mon Sep 17 00:00:00 2001 From: ephys Date: Sun, 12 Jun 2022 12:53:25 +0200 Subject: [PATCH 23/40] test(db2): fix db2 tests --- test/integration/model.test.js | 2 +- test/integration/query-interface.test.js | 21 ++++++++++++- test/integration/sequelize/query.test.js | 38 ++++++++++++++---------- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/test/integration/model.test.js b/test/integration/model.test.js index c0c5cc68f801..7c5342c428a6 100644 --- a/test/integration/model.test.js +++ b/test/integration/model.test.js @@ -2172,7 +2172,7 @@ describe(Support.getTestDialectTeaser('Model'), () => { const expectedLengths = { mssql: 2, postgres: 2, - db2: 10, + db2: 2, mariadb: 3, mysql: 1, sqlite: 1, diff --git a/test/integration/query-interface.test.js b/test/integration/query-interface.test.js index d4c5dc63b232..608a2e29d125 100644 --- a/test/integration/query-interface.test.js +++ b/test/integration/query-interface.test.js @@ -41,7 +41,16 @@ describe(Support.getTestDialectTeaser('QueryInterface'), () => { it('should not contain views', async function () { async function cleanup(sequelize) { if (dialect === 'db2') { - await sequelize.query('DROP VIEW V_Fail'); + // DB2 does not support DROP VIEW IF EXISTS + try { + await sequelize.query('DROP VIEW V_Fail'); + } catch (error) { + // -204 means V_Fail does not exist + // https://www.ibm.com/docs/en/db2-for-zos/11?topic=sec-204 + if (error.cause.sqlcode !== -204) { + throw error; + } + } } else { await sequelize.query('DROP VIEW IF EXISTS V_Fail'); } @@ -634,10 +643,20 @@ describe(Support.getTestDialectTeaser('QueryInterface'), () => { type: DataTypes.STRING, allowNull: false, }); + + // in db2, after the above "change column" query, the "users" table is in "reorg pending state" + // and the subsequent "add constraint" will fail. + // 'REORG TABLE' forces the reorg to happen now. + // https://www.ibm.com/support/pages/sql0668n-operating-not-allowed-reason-code-7-seen-when-querying-or-viewing-table-db2-warehouse-cloud-and-db2-cloud + if (dialect === 'db2') { + await this.sequelize.query(`CALL SYSPROC.ADMIN_CMD('REORG TABLE "users"')`); + } + await this.queryInterface.addConstraint('users', { type: 'PRIMARY KEY', fields: ['username'], }); + await this.queryInterface.addConstraint('posts', { fields: ['username'], references: { diff --git a/test/integration/sequelize/query.test.js b/test/integration/sequelize/query.test.js index 620501fd1674..c683d1ec31af 100644 --- a/test/integration/sequelize/query.test.js +++ b/test/integration/sequelize/query.test.js @@ -266,22 +266,28 @@ describe(Support.getTestDialectTeaser('Sequelize'), () => { })).to.include('john'); }); } else if (dialect === 'db2') { - it('executes stored procedures', function () { - const self = this; - - return self.sequelize.query(this.insertQuery).then(() => { - return self.sequelize.query('DROP PROCEDURE foo').then(() => { - return self.sequelize.query( - `CREATE PROCEDURE foo() DYNAMIC RESULT SETS 1 LANGUAGE SQL BEGIN DECLARE cr1 CURSOR WITH RETURN FOR SELECT * FROM ${qq(self.User.tableName)}; OPEN cr1; END`, - ).then(() => { - return self.sequelize.query('CALL foo()').then(users => { - expect(users.map(u => { - return u.username; - })).to.include('john'); - }); - }); - }); - }); + it('executes stored procedures', async function () { + const { sequelize } = this; + + await sequelize.query(this.insertQuery); + + try { + await sequelize.query('DROP PROCEDURE foo'); + } catch (error) { + // DB2 does not support DROP PROCEDURE IF EXISTS + // -204 means "FOO" does not exist + // https://www.ibm.com/docs/en/db2-for-zos/11?topic=sec-204 + if (error.cause.sqlcode !== -204) { + throw error; + } + } + + await sequelize.query( + `CREATE PROCEDURE foo() DYNAMIC RESULT SETS 1 LANGUAGE SQL BEGIN DECLARE cr1 CURSOR WITH RETURN FOR SELECT * FROM ${qq(this.User.tableName)}; OPEN cr1; END`, + ); + + const users = await sequelize.query('CALL foo()'); + expect(users.map(u => u.username)).to.include('john'); }); } else { console.log(': I want to be supported in this dialect as well :-('); From de8141d30d381c2caae4ab909bdd0bcf9d5a7d62 Mon Sep 17 00:00:00 2001 From: ephys Date: Sun, 12 Jun 2022 13:13:21 +0200 Subject: [PATCH 24/40] test(db2): fix broken test --- test/integration/query-interface.test.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/integration/query-interface.test.js b/test/integration/query-interface.test.js index 608a2e29d125..a27da19ca476 100644 --- a/test/integration/query-interface.test.js +++ b/test/integration/query-interface.test.js @@ -618,6 +618,15 @@ describe(Support.getTestDialectTeaser('QueryInterface'), () => { type: DataTypes.STRING, allowNull: false, }); + + // in db2, after the above "change column" query, the "users" table is in "reorg pending state" + // and the subsequent "add constraint" will fail. + // 'REORG TABLE' forces the reorg to happen now. + // https://www.ibm.com/support/pages/sql0668n-operating-not-allowed-reason-code-7-seen-when-querying-or-viewing-table-db2-warehouse-cloud-and-db2-cloud + if (dialect === 'db2') { + await this.sequelize.query(`CALL SYSPROC.ADMIN_CMD('REORG TABLE "users"')`); + } + await this.queryInterface.addConstraint('users', { fields: ['username'], type: 'PRIMARY KEY', From 7242c473afccfea4fd9002fbe963fccd51cd7abf Mon Sep 17 00:00:00 2001 From: ephys Date: Sun, 12 Jun 2022 13:14:33 +0200 Subject: [PATCH 25/40] refactor: clean up --- src/dialects/db2/query.js | 47 ------------------------------------- test/integration/support.js | 16 ------------- 2 files changed, 63 deletions(-) diff --git a/src/dialects/db2/query.js b/src/dialects/db2/query.js index 7d4d802c38cc..874a69c2bfee 100644 --- a/src/dialects/db2/query.js +++ b/src/dialects/db2/query.js @@ -196,53 +196,6 @@ export class Db2Query extends AbstractQuery { return null; } - // if (err.message.search('SQL0204N') !== -1 && _.startsWith(sql, 'DROP ')) { - // err = null; // Ignore table not found error for drop table. - // } else if (err.message.search('SQL0443N') !== -1) { - // if (this.isDropSchemaQuery()) { - // // Delete ERRORSCHEMA.ERRORTABLE if it exist. - // connection.querySync('DROP TABLE ERRORSCHEMA.ERRORTABLE;'); - // // Retry deleting the schema - // connection.querySync(this.sql); - // } - // - // err = null; // Ignore drop schema error. - // } else if (err.message.search('SQL0601N') !== -1) { - // const match = err.message.match(/SQL0601N {2}The name of the object to be created is identical to the existing name "(.*)" of type "(.*)"./); - // if (match && match.length > 1 && match[2] === 'TABLE') { - // let table; - // const mtarray = match[1].split('.'); - // if (mtarray[1]) { - // table = `"${mtarray[0]}"."${mtarray[1]}"`; - // } else { - // table = `"${mtarray[0]}"`; - // } - // - // if (connection.dropTable !== false) { - // connection.querySync(`DROP TABLE ${table}`); - // err = connection.querySync(sql); - // } else { - // err = null; - // } - // } else { - // err = null; // Ignore create schema error. - // } - // } else if (err.message.search('SQL0911N') !== -1) { - // if (err.message.search('Reason code "2"') !== -1) { - // err = null; // Ignore deadlock error due to program logic. - // } - // } else if (err.message.search('SQL0605W') !== -1) { - // err = null; // Ignore warning. - // } else if (err.message.search('SQL0668N') !== -1 - // && _.startsWith(sql, 'ALTER TABLE ')) { - // connection.querySync(`CALL SYSPROC.ADMIN_CMD('REORG TABLE ${sql.slice(12).split(' ')[0]}')`); - // err = connection.querySync(sql); - // } - // - // if (err && err.length === 0) { - // err = null; - // } - // return err; } diff --git a/test/integration/support.js b/test/integration/support.js index d0008683c6b9..1af1d0454319 100644 --- a/test/integration/support.js +++ b/test/integration/support.js @@ -6,28 +6,12 @@ const { setTimeout, clearTimeout } = global; const pTimeout = require('p-timeout'); const Support = require('../support'); -const { getTestDialect } = require('../support'); const CLEANUP_TIMEOUT = Number.parseInt(process.env.SEQ_TEST_CLEANUP_TIMEOUT, 10) || 10_000; let runningQueries = new Set(); before(async function () { - if (getTestDialect() === 'db2') { - // needed by dropSchema function - await this.sequelize.query(` - CREATE TABLESPACE SYSTOOLSPACE IN IBMCATGROUP - MANAGED BY AUTOMATIC STORAGE USING STOGROUP IBMSTOGROUP - EXTENTSIZE 4; - `); - - await this.sequelize.query(` - CREATE USER TEMPORARY TABLESPACE SYSTOOLSTMPSPACE IN IBMCATGROUP - MANAGED BY AUTOMATIC STORAGE USING STOGROUP IBMSTOGROUP - EXTENTSIZE 4 - `); - } - this.sequelize.addHook('beforeQuery', (options, query) => { runningQueries.add(query); }); From 74061bd1d1f1b7f6837ab4592d31e29206851db4 Mon Sep 17 00:00:00 2001 From: ephys Date: Sun, 12 Jun 2022 13:18:39 +0200 Subject: [PATCH 26/40] refactor: revert unnecessary change --- test/integration/support.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/support.js b/test/integration/support.js index 1af1d0454319..969e7de39c5f 100644 --- a/test/integration/support.js +++ b/test/integration/support.js @@ -11,7 +11,7 @@ const CLEANUP_TIMEOUT = Number.parseInt(process.env.SEQ_TEST_CLEANUP_TIMEOUT, 10 let runningQueries = new Set(); -before(async function () { +before(function () { this.sequelize.addHook('beforeQuery', (options, query) => { runningQueries.add(query); }); From 6e29070185202f871208666b88ff5ea2572a5ae6 Mon Sep 17 00:00:00 2001 From: ephys Date: Sun, 12 Jun 2022 13:30:15 +0200 Subject: [PATCH 27/40] test(db2): bring back systoolspace creation --- src/dialects/db2/query.js | 2 +- test/integration/support.js | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/dialects/db2/query.js b/src/dialects/db2/query.js index 874a69c2bfee..793268b997b0 100644 --- a/src/dialects/db2/query.js +++ b/src/dialects/db2/query.js @@ -191,7 +191,7 @@ export class Db2Query extends AbstractQuery { filterSQLError(err, sql, connection) { // This error is safe to ignore: - // [IBM][CLI Driver][DB2/LINUXX8664] SQL0605W The index was not created because an index "x" with a matching definition already exists. SQLSTATE=01550 + // [IBM][CLI Driver][DB2/LINUXX8664] SQL0605W The index was not created because an index "x" with a matching definition already exists. SQLSTATE=01550 if (err.message.search('SQL0605W') !== -1) { return null; } diff --git a/test/integration/support.js b/test/integration/support.js index 969e7de39c5f..d0008683c6b9 100644 --- a/test/integration/support.js +++ b/test/integration/support.js @@ -6,12 +6,28 @@ const { setTimeout, clearTimeout } = global; const pTimeout = require('p-timeout'); const Support = require('../support'); +const { getTestDialect } = require('../support'); const CLEANUP_TIMEOUT = Number.parseInt(process.env.SEQ_TEST_CLEANUP_TIMEOUT, 10) || 10_000; let runningQueries = new Set(); -before(function () { +before(async function () { + if (getTestDialect() === 'db2') { + // needed by dropSchema function + await this.sequelize.query(` + CREATE TABLESPACE SYSTOOLSPACE IN IBMCATGROUP + MANAGED BY AUTOMATIC STORAGE USING STOGROUP IBMSTOGROUP + EXTENTSIZE 4; + `); + + await this.sequelize.query(` + CREATE USER TEMPORARY TABLESPACE SYSTOOLSTMPSPACE IN IBMCATGROUP + MANAGED BY AUTOMATIC STORAGE USING STOGROUP IBMSTOGROUP + EXTENTSIZE 4 + `); + } + this.sequelize.addHook('beforeQuery', (options, query) => { runningQueries.add(query); }); From d162544ad435b691c6de78b059bca342ac3293a9 Mon Sep 17 00:00:00 2001 From: ephys Date: Mon, 13 Jun 2022 19:34:31 +0200 Subject: [PATCH 28/40] fix(db2): specify schema in create index --- src/dialects/abstract/query-generator.js | 23 ++++--- test/integration/model/schema.test.js | 67 -------------------- test/integration/model/sync.test.js | 81 +++++++++++++++++++++++- 3 files changed, 95 insertions(+), 76 deletions(-) diff --git a/src/dialects/abstract/query-generator.js b/src/dialects/abstract/query-generator.js index 97a5535adee2..91abc63445d9 100644 --- a/src/dialects/abstract/query-generator.js +++ b/src/dialects/abstract/query-generator.js @@ -613,18 +613,14 @@ export class AbstractQueryGenerator { options.where = this.whereQuery(options.where); } - if (typeof tableName === 'string') { - tableName = this.quoteIdentifiers(tableName); - } else { - tableName = this.quoteTable(tableName); - } + const escapedTableName = typeof tableName === 'string' ? this.quoteIdentifiers(tableName) : this.quoteTable(tableName); const concurrently = this._dialect.supports.index.concurrently && options.concurrently ? 'CONCURRENTLY' : undefined; let ind; if (this._dialect.supports.indexViaAlter) { ind = [ 'ALTER TABLE', - tableName, + escapedTableName, concurrently, 'ADD', ]; @@ -632,13 +628,24 @@ export class AbstractQueryGenerator { ind = ['CREATE']; } + // DB2 incorrectly scopes the index if we don't specify the schema name, + // which will cause it to error if another schema contains a table that uses an index with an identical name + const escapedIndexName = tableName.schema && this.dialect === 'db2' + // 'quoteTable' isn't the best name: it quotes any identifier. + // in this case, the goal is to produce '"schema_name"."index_name"' to scope the index in this schema + ? this.quoteTable({ + schema: tableName.schema, + tableName: options.name, + }) + : this.quoteIdentifiers(options.name); + ind = ind.concat( options.unique ? 'UNIQUE' : '', options.type, 'INDEX', !this._dialect.supports.indexViaAlter ? concurrently : undefined, - this.quoteIdentifiers(options.name), + escapedIndexName, this._dialect.supports.index.using === 1 && options.using ? `USING ${options.using}` : '', - !this._dialect.supports.indexViaAlter ? `ON ${tableName}` : undefined, + !this._dialect.supports.indexViaAlter ? `ON ${escapedTableName}` : undefined, this._dialect.supports.index.using === 2 && options.using ? `USING ${options.using}` : '', `(${fieldsSql.join(', ')})`, this._dialect.supports.index.parser && options.parser ? `WITH PARSER ${options.parser}` : undefined, diff --git a/test/integration/model/schema.test.js b/test/integration/model/schema.test.js index 90fe9292d79f..863930f9f81b 100644 --- a/test/integration/model/schema.test.js +++ b/test/integration/model/schema.test.js @@ -481,73 +481,6 @@ describe(Support.getTestDialectTeaser('Model'), () => { } }); }); - - describe('regressions', () => { - it('should be able to sync model with schema', async function () { - const User = this.sequelize.define('User1', { - name: DataTypes.STRING, - value: DataTypes.INTEGER, - }, { - schema: SCHEMA_ONE, - indexes: [ - { - name: 'test_slug_idx', - fields: ['name'], - }, - ], - }); - - const Task = this.sequelize.define('Task2', { - name: DataTypes.STRING, - value: DataTypes.INTEGER, - }, { - schema: SCHEMA_TWO, - indexes: [ - { - name: 'test_slug_idx', - fields: ['name'], - }, - ], - }); - - await User.sync({ force: true }); - await Task.sync({ force: true }); - - const [user, task] = await Promise.all([ - this.sequelize.queryInterface.describeTable(User.tableName, SCHEMA_ONE), - this.sequelize.queryInterface.describeTable(Task.tableName, SCHEMA_TWO), - ]); - - expect(user).to.be.ok; - expect(task).to.be.ok; - }); - - // TODO: this should work with MSSQL / MariaDB too - // Need to fix addSchema return type - if (dialect.startsWith('postgres')) { - it('defaults to schema provided to sync() for references #11276', async function () { - const User = this.sequelize.define('UserXYZ', { - uid: { - type: DataTypes.INTEGER, - primaryKey: true, - autoIncrement: true, - allowNull: false, - }, - }); - const Task = this.sequelize.define('TaskXYZ', {}); - - Task.belongsTo(User); - - await User.sync({ force: true, schema: SCHEMA_ONE }); - await Task.sync({ force: true, schema: SCHEMA_ONE }); - const user0 = await User.schema(SCHEMA_ONE).create({}); - const task = await Task.schema(SCHEMA_ONE).create({}); - await task.setUserXYZ(user0); - const user = await task.getUserXYZ({ schema: SCHEMA_ONE }); - expect(user).to.be.ok; - }); - } - }); }); } }); diff --git a/test/integration/model/sync.test.js b/test/integration/model/sync.test.js index dc4f8f73fc10..752c0a10acbb 100644 --- a/test/integration/model/sync.test.js +++ b/test/integration/model/sync.test.js @@ -6,7 +6,7 @@ const { sequelize, getTestDialect, getTestDialectTeaser } = require('../support' const dialect = getTestDialect(); -describe(getTestDialectTeaser('Model.sync & Sequelize.sync'), () => { +describe(getTestDialectTeaser('Model.sync & Sequelize#sync'), () => { it('removes a column if it exists in the databases schema but not the model', async () => { const User = sequelize.define('testSync', { name: DataTypes.STRING, @@ -423,6 +423,85 @@ describe(getTestDialectTeaser('Model.sync & Sequelize.sync'), () => { expect(getIndexFields(out2[0])).to.deep.eq(['email']); expect(out2[0].unique).to.eq(true, 'index should not be unique'); }); + + const SCHEMA_ONE = 'schema_one'; + const SCHEMA_TWO = 'schema_two'; + + it('can create two identically named indexes in different schemas', async function () { + await Promise.all([ + sequelize.createSchema(SCHEMA_ONE), + sequelize.createSchema(SCHEMA_TWO), + ]); + + const User = this.sequelize.define('User1', { + name: DataTypes.STRING, + }, { + schema: SCHEMA_ONE, + indexes: [ + { + name: 'test_slug_idx', + fields: ['name'], + }, + ], + }); + + const Task = this.sequelize.define('Task2', { + name: DataTypes.STRING, + }, { + schema: SCHEMA_TWO, + indexes: [ + { + name: 'test_slug_idx', + fields: ['name'], + }, + ], + }); + + await User.sync({ force: true }); + await Task.sync({ force: true }); + + const [user, task] = await Promise.all([ + this.sequelize.queryInterface.describeTable(User.tableName, SCHEMA_ONE), + this.sequelize.queryInterface.describeTable(Task.tableName, SCHEMA_TWO), + ]); + + expect(user).to.be.ok; + expect(task).to.be.ok; + }); + + // TODO: this should work with MSSQL / MariaDB too + // Need to fix addSchema return type + if (dialect.startsWith('postgres')) { + it('defaults to schema provided to sync() for references #11276', async function () { + await Promise.all([ + sequelize.createSchema(SCHEMA_ONE), + sequelize.createSchema(SCHEMA_TWO), + ]); + + const User = this.sequelize.define('UserXYZ', { + uid: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + }, + }); + const Task = this.sequelize.define('TaskXYZ', {}); + + Task.belongsTo(User); + + // TODO: do we really want to keep this? Shouldn't model schemas be defined and fixed? + await User.sync({ force: true, schema: SCHEMA_ONE }); + await Task.sync({ force: true, schema: SCHEMA_ONE }); + const user0 = await User.withSchema(SCHEMA_ONE).create({}); + const task = await Task.withSchema(SCHEMA_ONE).create({}); + await task.setUserXYZ(user0); + + // TODO: do we really want to keep this? Shouldn't model schemas be defined and fixed? + const user = await task.getUserXYZ({ schema: SCHEMA_ONE }); + expect(user).to.be.ok; + }); + } }); async function getNonPrimaryIndexes(model) { From c425668f2e25da26f362cbb25f45ed59f75dbab8 Mon Sep 17 00:00:00 2001 From: ephys Date: Mon, 13 Jun 2022 19:44:29 +0200 Subject: [PATCH 29/40] test(db2): update tests --- test/unit/sql/index.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/sql/index.test.js b/test/unit/sql/index.test.js index 754ec1c7b760..131bd28cf091 100644 --- a/test/unit/sql/index.test.js +++ b/test/unit/sql/index.test.js @@ -33,6 +33,7 @@ describe(Support.getTestDialectTeaser('SQL'), () => { tableName: 'table', }, ['column1', 'column2'], {}, 'schema_table'), { default: 'CREATE INDEX [schema_table_column1_column2] ON [schema].[table] ([column1], [column2])', + db2: 'CREATE INDEX "schema"."schema_table_column1_column2" ON "schema"."table" ("column1", "column2")', mariadb: 'ALTER TABLE `schema`.`table` ADD INDEX `schema_table_column1_column2` (`column1`, `column2`)', }); From 1d53695ea59eff67ec3cbefd2c037ac4d4b9d4b7 Mon Sep 17 00:00:00 2001 From: ephys Date: Mon, 13 Jun 2022 20:08:30 +0200 Subject: [PATCH 30/40] fix(sqlite): scope index name --- src/dialects/abstract/query-generator.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dialects/abstract/query-generator.js b/src/dialects/abstract/query-generator.js index 91abc63445d9..6535fc6b5038 100644 --- a/src/dialects/abstract/query-generator.js +++ b/src/dialects/abstract/query-generator.js @@ -628,9 +628,9 @@ export class AbstractQueryGenerator { ind = ['CREATE']; } - // DB2 incorrectly scopes the index if we don't specify the schema name, + // DB2 & SQLite incorrectly scopes the index if we don't specify the schema name, // which will cause it to error if another schema contains a table that uses an index with an identical name - const escapedIndexName = tableName.schema && this.dialect === 'db2' + const escapedIndexName = tableName.schema && (this.dialect === 'db2' || this.dialect === 'sqlite') // 'quoteTable' isn't the best name: it quotes any identifier. // in this case, the goal is to produce '"schema_name"."index_name"' to scope the index in this schema ? this.quoteTable({ From cf1008528950cb960497bf2e1ce7f86ba25b2ba5 Mon Sep 17 00:00:00 2001 From: ephys Date: Mon, 13 Jun 2022 20:47:24 +0200 Subject: [PATCH 31/40] fix(sqlite): rollback change --- src/dialects/abstract/query-generator.js | 4 +- test/integration/model/sync.test.js | 77 +++++++++++++----------- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/src/dialects/abstract/query-generator.js b/src/dialects/abstract/query-generator.js index 6535fc6b5038..91abc63445d9 100644 --- a/src/dialects/abstract/query-generator.js +++ b/src/dialects/abstract/query-generator.js @@ -628,9 +628,9 @@ export class AbstractQueryGenerator { ind = ['CREATE']; } - // DB2 & SQLite incorrectly scopes the index if we don't specify the schema name, + // DB2 incorrectly scopes the index if we don't specify the schema name, // which will cause it to error if another schema contains a table that uses an index with an identical name - const escapedIndexName = tableName.schema && (this.dialect === 'db2' || this.dialect === 'sqlite') + const escapedIndexName = tableName.schema && this.dialect === 'db2' // 'quoteTable' isn't the best name: it quotes any identifier. // in this case, the goal is to produce '"schema_name"."index_name"' to scope the index in this schema ? this.quoteTable({ diff --git a/test/integration/model/sync.test.js b/test/integration/model/sync.test.js index 752c0a10acbb..283048988e89 100644 --- a/test/integration/model/sync.test.js +++ b/test/integration/model/sync.test.js @@ -427,47 +427,52 @@ describe(getTestDialectTeaser('Model.sync & Sequelize#sync'), () => { const SCHEMA_ONE = 'schema_one'; const SCHEMA_TWO = 'schema_two'; - it('can create two identically named indexes in different schemas', async function () { - await Promise.all([ - sequelize.createSchema(SCHEMA_ONE), - sequelize.createSchema(SCHEMA_TWO), - ]); + if (sequelize.dialect.supports.schemas) { + it('can create two identically named indexes in different schemas', async () => { + await Promise.all([ + sequelize.createSchema(SCHEMA_ONE), + sequelize.createSchema(SCHEMA_TWO), + ]); - const User = this.sequelize.define('User1', { - name: DataTypes.STRING, - }, { - schema: SCHEMA_ONE, - indexes: [ - { - name: 'test_slug_idx', - fields: ['name'], - }, - ], - }); + const User = sequelize.define('User1', { + name: DataTypes.STRING, + }, { + schema: SCHEMA_ONE, + indexes: [ + { + name: 'test_slug_idx', + fields: ['name'], + }, + ], + }); - const Task = this.sequelize.define('Task2', { - name: DataTypes.STRING, - }, { - schema: SCHEMA_TWO, - indexes: [ - { - name: 'test_slug_idx', - fields: ['name'], - }, - ], - }); + const Task = sequelize.define('Task2', { + name: DataTypes.STRING, + }, { + schema: SCHEMA_TWO, + indexes: [ + { + name: 'test_slug_idx', + fields: ['name'], + }, + ], + }); - await User.sync({ force: true }); - await Task.sync({ force: true }); + await User.sync({ force: true }); + await Task.sync({ force: true }); + + const [userIndexes, taskIndexes] = await Promise.all([ + getNonPrimaryIndexes(User), + getNonPrimaryIndexes(Task), + ]); - const [user, task] = await Promise.all([ - this.sequelize.queryInterface.describeTable(User.tableName, SCHEMA_ONE), - this.sequelize.queryInterface.describeTable(Task.tableName, SCHEMA_TWO), - ]); + expect(userIndexes).to.have.length(1); + expect(taskIndexes).to.have.length(1); - expect(user).to.be.ok; - expect(task).to.be.ok; - }); + expect(userIndexes[0].name).to.eq('test_slug_idx'); + expect(taskIndexes[0].name).to.eq('test_slug_idx'); + }); + } // TODO: this should work with MSSQL / MariaDB too // Need to fix addSchema return type From 30c3566f0cd502f1ea347ca976df3a1d4a9b7197 Mon Sep 17 00:00:00 2001 From: ephys Date: Mon, 13 Jun 2022 20:51:53 +0200 Subject: [PATCH 32/40] test(db2): dont create tablespace if it exists --- test/integration/support.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/integration/support.js b/test/integration/support.js index d0008683c6b9..ba7b3ff05375 100644 --- a/test/integration/support.js +++ b/test/integration/support.js @@ -4,6 +4,7 @@ // avoiding to be affected unintentionally by `sinon.useFakeTimers()` called by the tests themselves. const { setTimeout, clearTimeout } = global; +const { QueryTypes } = require('@sequelize/core'); const pTimeout = require('p-timeout'); const Support = require('../support'); const { getTestDialect } = require('../support'); @@ -14,18 +15,26 @@ let runningQueries = new Set(); before(async function () { if (getTestDialect() === 'db2') { - // needed by dropSchema function - await this.sequelize.query(` + const res = await this.sequelize.query(`SELECT TBSPACE FROM SYSCAT.TABLESPACES WHERE TBSPACE = 'SYSTOOLSPACE'`, { + type: QueryTypes.SELECT, + }); + + const tableExists = res[0].TBSPACE === 'SYSTOOLSPACE'; + + if (!tableExists) { + // needed by dropSchema function + await this.sequelize.query(` CREATE TABLESPACE SYSTOOLSPACE IN IBMCATGROUP MANAGED BY AUTOMATIC STORAGE USING STOGROUP IBMSTOGROUP EXTENTSIZE 4; `); - await this.sequelize.query(` + await this.sequelize.query(` CREATE USER TEMPORARY TABLESPACE SYSTOOLSTMPSPACE IN IBMCATGROUP MANAGED BY AUTOMATIC STORAGE USING STOGROUP IBMSTOGROUP EXTENTSIZE 4 `); + } } this.sequelize.addHook('beforeQuery', (options, query) => { From 6dcc25ddd152e97670aa4e6df04ffe44c542a153 Mon Sep 17 00:00:00 2001 From: ephys Date: Mon, 13 Jun 2022 21:06:55 +0200 Subject: [PATCH 33/40] test(db2): damnit --- test/integration/support.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/support.js b/test/integration/support.js index ba7b3ff05375..d36a0cdb2600 100644 --- a/test/integration/support.js +++ b/test/integration/support.js @@ -19,7 +19,7 @@ before(async function () { type: QueryTypes.SELECT, }); - const tableExists = res[0].TBSPACE === 'SYSTOOLSPACE'; + const tableExists = res[0]?.TBSPACE === 'SYSTOOLSPACE'; if (!tableExists) { // needed by dropSchema function From 3d72110f015374143bbed452e5d4b05bbcc00c31 Mon Sep 17 00:00:00 2001 From: ephys Date: Mon, 13 Jun 2022 23:10:38 +0200 Subject: [PATCH 34/40] feat: move the deletion of error table after drop schema --- src/dialects/db2/query-generator.js | 6 ++-- src/dialects/db2/query-interface.js | 48 +++++++++++++++++++++++++++-- src/dialects/db2/query.js | 13 ++++++++ src/sequelize.js | 5 ++- 4 files changed, 65 insertions(+), 7 deletions(-) diff --git a/src/dialects/db2/query-generator.js b/src/dialects/db2/query-generator.js index 79decb8727f0..476df101d1a3 100644 --- a/src/dialects/db2/query-generator.js +++ b/src/dialects/db2/query-generator.js @@ -37,13 +37,13 @@ export class Db2QueryGenerator extends AbstractQueryGenerator { // DROP SCHEMA Can't drop schema if it is not empty. // DROP SCHEMA Can't drop objects belonging to the schema // So, call the admin procedure to drop schema. - const query = `CALL SYSPROC.ADMIN_DROP_SCHEMA(${wrapSingleQuote(schema.trim())}, NULL, $sequelize_1, $sequelize_2)`; + const query = `CALL SYSPROC.ADMIN_DROP_SCHEMA(${wrapSingleQuote(schema.trim())}, NULL, $sequelize_errorSchema, $sequelize_errorTable)`; return { query, bind: { - sequelize_1: { ParamType: 'INOUT', Data: 'ERRORSCHEMA' }, - sequelize_2: { ParamType: 'INOUT', Data: 'ERRORTABLE' }, + sequelize_errorSchema: { ParamType: 'INOUT', Data: 'ERRORSCHEMA' }, + sequelize_errorTable: { ParamType: 'INOUT', Data: `ERRORTABLE` }, }, }; } diff --git a/src/dialects/db2/query-interface.js b/src/dialects/db2/query-interface.js index 105e6d1cf20b..933b8b035ca5 100644 --- a/src/dialects/db2/query-interface.js +++ b/src/dialects/db2/query-interface.js @@ -1,5 +1,6 @@ 'use strict'; +import { AggregateError, DatabaseError } from '../../errors'; import { assertNoReservedBind } from '../../utils/sql'; const _ = require('lodash'); @@ -85,10 +86,51 @@ export class Db2QueryInterface extends QueryInterface { } async dropSchema(schema, options) { - // error from dropSchema will be stored in this table. - await this.dropTable({ tableName: 'ERRORTABLE', schema: 'ERRORSCHEMA' }); + const outParams = new Map(); - return super.dropSchema(schema, options); + // DROP SCHEMA works in a weird way in DB2: + // Its query uses ADMIN_DROP_SCHEMA, which stores the error message in a table + // specified by two IN-OUT parameters. + // If the returned values for these parameters is not null, then an error occurred. + const response = await super.dropSchema(schema, { + ...options, + // db2 supports out parameters. We don't have a proper API for it yet + // so this temporary API will have to do. + _unsafe_db2Outparams: outParams, + }); + + const errorTable = outParams.get('sequelize_errorTable'); + if (errorTable != null) { + const errorSchema = outParams.get('sequelize_errorSchema'); + + const errorData = await this.sequelize.queryRaw(`SELECT * FROM "${errorSchema}"."${errorTable}"`, { + type: QueryTypes.SELECT, + }); + + // replicate the data ibm_db adds on an error object + const error = new Error(errorData[0].DIAGTEXT); + error.sqlcode = errorData[0].SQLCODE; + error.sql = errorData[0].STATEMENT; + error.state = errorData[0].sqlstate; + + const wrappedError = new DatabaseError(error); + + try { + await this.dropTable({ + tableName: errorTable, + schema: errorSchema, + }); + } catch (dropError) { + throw new AggregateError([ + wrappedError, + new Error(`An error occurred while cleaning up table ${errorSchema}.${errorTable}`, { cause: dropError }), + ]); + } + + throw wrappedError; + } + + return response; } async createTable(tableName, attributes, options, model) { diff --git a/src/dialects/db2/query.js b/src/dialects/db2/query.js index 793268b997b0..61c291030e6e 100644 --- a/src/dialects/db2/query.js +++ b/src/dialects/db2/query.js @@ -95,6 +95,8 @@ export class Db2Query extends AbstractQuery { const SQL = this.sql.toUpperCase(); let newSql = this.sql; + + // TODO: move this to Db2QueryGenerator if ((this.isSelectQuery() || _.startsWith(SQL, 'SELECT ')) && !SQL.includes(' FROM ', 8)) { if (this.sql.charAt(this.sql.length - 1) === ';') { @@ -112,6 +114,17 @@ export class Db2Query extends AbstractQuery { stmt.execute(params, (err, result, outparams) => { debug(`executed(${this.connection.uuid || 'default'}):${newSql} ${parameters ? util.inspect(parameters, { compact: true, breakLength: Infinity }) : ''}`); + // map the INOUT parameters to the name provided by the dev + // this is an internal API, not yet ready for dev consumption, hence the _unsafe_ prefix. + if (outparams && this.options.bindParameterOrder && this.options._unsafe_db2Outparams) { + for (let i = 0; i < this.options.bindParameterOrder.length; i++) { + const paramName = this.options.bindParameterOrder[i]; + const paramValue = outparams[i]; + + this.options._unsafe_db2Outparams.set(paramName, paramValue); + } + } + if (benchmark) { this.sequelize.log(`Executed (${this.connection.uuid || 'default'}): ${newSql} ${parameters ? util.inspect(parameters, { compact: true, breakLength: Infinity }) : ''}`, Date.now() - queryBegin, this.options); } diff --git a/src/sequelize.js b/src/sequelize.js index 08a55a5a5491..c0d4bf8f365e 100644 --- a/src/sequelize.js +++ b/src/sequelize.js @@ -526,9 +526,10 @@ Only bind parameters can be provided, in the dialect-specific syntax. Use Sequelize#query if you wish to use replacements.`); } - options = { ...this.options.query, ...options }; + options = { ...this.options.query, ...options, bindParameterOrder: null }; let bindParameters; + let bindParameterOrder; if (options.bind != null) { const isBindArray = Array.isArray(options.bind); if (!isPlainObject(options.bind) && !isBindArray) { @@ -549,6 +550,8 @@ Use Sequelize#query if you wish to use replacements.`); sql = mappedResult.sql; + // used by dialects that support "INOUT" parameters to map the OUT parameters back the the name the dev used. + options.bindParameterOrder = mappedResult.bindOrder; if (mappedResult.bindOrder == null) { bindParameters = options.bind; } else { From 04c422a44379aea5c542a041cbc3a4079cd643d4 Mon Sep 17 00:00:00 2001 From: ephys Date: Mon, 13 Jun 2022 23:19:28 +0200 Subject: [PATCH 35/40] fix(db2): reduce risk of error table collisions --- src/dialects/db2/query-generator.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/dialects/db2/query-generator.js b/src/dialects/db2/query-generator.js index 476df101d1a3..efbc969653a1 100644 --- a/src/dialects/db2/query-generator.js +++ b/src/dialects/db2/query-generator.js @@ -33,17 +33,23 @@ export class Db2QueryGenerator extends AbstractQueryGenerator { ].join(' '); } + _errorTableCount = 0; + dropSchema(schema) { // DROP SCHEMA Can't drop schema if it is not empty. // DROP SCHEMA Can't drop objects belonging to the schema // So, call the admin procedure to drop schema. const query = `CALL SYSPROC.ADMIN_DROP_SCHEMA(${wrapSingleQuote(schema.trim())}, NULL, $sequelize_errorSchema, $sequelize_errorTable)`; + if (this._errorTableCount >= Number.MAX_SAFE_INTEGER) { + this._errorTableCount = 0; + } + return { query, bind: { sequelize_errorSchema: { ParamType: 'INOUT', Data: 'ERRORSCHEMA' }, - sequelize_errorTable: { ParamType: 'INOUT', Data: `ERRORTABLE` }, + sequelize_errorTable: { ParamType: 'INOUT', Data: `ERRORTABLE${this._errorTableCount++}` }, }, }; } From c20d44bb1778fd6750fe676d048dc136efc7304d Mon Sep 17 00:00:00 2001 From: ephys Date: Mon, 13 Jun 2022 23:25:01 +0200 Subject: [PATCH 36/40] feat: re-add forced table reorg --- src/dialects/db2/query-interface.js | 29 ++++++++++++++++++++++++ test/integration/query-interface.test.js | 16 ------------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/dialects/db2/query-interface.js b/src/dialects/db2/query-interface.js index 933b8b035ca5..ed2f1cc1256f 100644 --- a/src/dialects/db2/query-interface.js +++ b/src/dialects/db2/query-interface.js @@ -171,4 +171,33 @@ export class Db2QueryInterface extends QueryInterface { return await this.sequelize.queryRaw(sql, options); } + async addConstraint(tableName, options) { + try { + return await super.addConstraint(tableName, options); + } catch (error) { + if (!error.cause) { + throw error; + } + + // Operation not allowed for reason code "7" on table "DB2INST1.users". SQLSTATE=57007 + if (error.cause.sqlcode !== -668 || error.cause.state !== '57007') { + throw error; + } + + // https://www.ibm.com/support/pages/how-verify-and-resolve-sql0668n-reason-code-7-when-accessing-table + await this.executeTableReorg(); + await super.addConstraint(tableName, options); + } + } + + /** + * DB2 can put tables in the "reorg pending" state after a structure change (e.g. ALTER) + * Other changes cannot be done to these tables until the reorg has been completed. + * + * This method forces a reorg to happen now. + */ + async executeTableReorg() { + // https://www.ibm.com/support/pages/sql0668n-operating-not-allowed-reason-code-7-seen-when-querying-or-viewing-table-db2-warehouse-cloud-and-db2-cloud + return await this.sequelize.query(`CALL SYSPROC.ADMIN_CMD('REORG TABLE "users"')`); + } } diff --git a/test/integration/query-interface.test.js b/test/integration/query-interface.test.js index a27da19ca476..4f1a2a55c6fc 100644 --- a/test/integration/query-interface.test.js +++ b/test/integration/query-interface.test.js @@ -619,14 +619,6 @@ describe(Support.getTestDialectTeaser('QueryInterface'), () => { allowNull: false, }); - // in db2, after the above "change column" query, the "users" table is in "reorg pending state" - // and the subsequent "add constraint" will fail. - // 'REORG TABLE' forces the reorg to happen now. - // https://www.ibm.com/support/pages/sql0668n-operating-not-allowed-reason-code-7-seen-when-querying-or-viewing-table-db2-warehouse-cloud-and-db2-cloud - if (dialect === 'db2') { - await this.sequelize.query(`CALL SYSPROC.ADMIN_CMD('REORG TABLE "users"')`); - } - await this.queryInterface.addConstraint('users', { fields: ['username'], type: 'PRIMARY KEY', @@ -653,14 +645,6 @@ describe(Support.getTestDialectTeaser('QueryInterface'), () => { allowNull: false, }); - // in db2, after the above "change column" query, the "users" table is in "reorg pending state" - // and the subsequent "add constraint" will fail. - // 'REORG TABLE' forces the reorg to happen now. - // https://www.ibm.com/support/pages/sql0668n-operating-not-allowed-reason-code-7-seen-when-querying-or-viewing-table-db2-warehouse-cloud-and-db2-cloud - if (dialect === 'db2') { - await this.sequelize.query(`CALL SYSPROC.ADMIN_CMD('REORG TABLE "users"')`); - } - await this.queryInterface.addConstraint('users', { type: 'PRIMARY KEY', fields: ['username'], From fbebfe0080cc941ae84b02ab3b7654e9386e37d9 Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 14 Jun 2022 20:15:45 +0200 Subject: [PATCH 37/40] fix(db2): make dropSchema "IF EXISTS" --- src/dialects/db2/query-interface.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/dialects/db2/query-interface.js b/src/dialects/db2/query-interface.js index ed2f1cc1256f..c33f329eaea2 100644 --- a/src/dialects/db2/query-interface.js +++ b/src/dialects/db2/query-interface.js @@ -111,7 +111,7 @@ export class Db2QueryInterface extends QueryInterface { const error = new Error(errorData[0].DIAGTEXT); error.sqlcode = errorData[0].SQLCODE; error.sql = errorData[0].STATEMENT; - error.state = errorData[0].sqlstate; + error.state = errorData[0].SQLSTATE; const wrappedError = new DatabaseError(error); @@ -127,6 +127,13 @@ export class Db2QueryInterface extends QueryInterface { ]); } + // -204 is "name is undefined" (schema does not exist) + // 'queryInterface.dropSchema' is supposed to be DROP SCHEMA IF EXISTS + // so we can ignore this error + if (error.sqlcode === -204 && error.state === '42704') { + return response; + } + throw wrappedError; } From 3bc66e773b54a931ae47a07cc0108fc4c061fd7d Mon Sep 17 00:00:00 2001 From: ephys Date: Tue, 14 Jun 2022 20:34:02 +0200 Subject: [PATCH 38/40] test(db2): prevent drop schema deadlock --- test/support.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/support.ts b/test/support.ts index 0e54bd726e08..ff0fd665d784 100644 --- a/test/support.ts +++ b/test/support.ts @@ -224,7 +224,16 @@ export async function dropTestSchemas(sequelize: Sequelize) { // @ts-expect-error const schemaName = schema.name ? schema.name : schema; if (schemaName !== sequelize.config.database) { - schemasPromise.push(sequelize.dropSchema(schemaName)); + const promise = sequelize.dropSchema(schemaName); + + if (getTestDialect() === 'db2') { + // https://github.com/sequelize/sequelize/pull/14453#issuecomment-1155581572 + // DB2 can sometimes deadlock / timeout when deleting more than one schema at the same time. + // eslint-disable-next-line no-await-in-loop + await promise; + } else { + schemasPromise.push(promise); + } } } From 168d2319e6ad96af3d28b982a326a08e5f3f20d1 Mon Sep 17 00:00:00 2001 From: ephys Date: Wed, 15 Jun 2022 10:02:00 +0200 Subject: [PATCH 39/40] fix: fix copy paste mistake --- src/dialects/db2/query-interface.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/dialects/db2/query-interface.js b/src/dialects/db2/query-interface.js index c33f329eaea2..7b7048575ffe 100644 --- a/src/dialects/db2/query-interface.js +++ b/src/dialects/db2/query-interface.js @@ -94,8 +94,8 @@ export class Db2QueryInterface extends QueryInterface { // If the returned values for these parameters is not null, then an error occurred. const response = await super.dropSchema(schema, { ...options, - // db2 supports out parameters. We don't have a proper API for it yet - // so this temporary API will have to do. + // TODO: db2 supports out parameters. We don't have a proper API for it yet + // for now, this temporary API will have to do. _unsafe_db2Outparams: outParams, }); @@ -192,7 +192,7 @@ export class Db2QueryInterface extends QueryInterface { } // https://www.ibm.com/support/pages/how-verify-and-resolve-sql0668n-reason-code-7-when-accessing-table - await this.executeTableReorg(); + await this.executeTableReorg(tableName); await super.addConstraint(tableName, options); } } @@ -202,9 +202,11 @@ export class Db2QueryInterface extends QueryInterface { * Other changes cannot be done to these tables until the reorg has been completed. * * This method forces a reorg to happen now. + * + * @param {TableName} tableName - The name of the table to reorg */ - async executeTableReorg() { + async executeTableReorg(tableName) { // https://www.ibm.com/support/pages/sql0668n-operating-not-allowed-reason-code-7-seen-when-querying-or-viewing-table-db2-warehouse-cloud-and-db2-cloud - return await this.sequelize.query(`CALL SYSPROC.ADMIN_CMD('REORG TABLE "users"')`); + return await this.sequelize.query(`CALL SYSPROC.ADMIN_CMD('REORG TABLE ${this.queryGenerator.quoteTable(tableName)}')`); } } From 10b37b2d2c68505c26b31fd86ad4a0bb17f3f2dc Mon Sep 17 00:00:00 2001 From: ephys Date: Wed, 15 Jun 2022 12:49:57 +0200 Subject: [PATCH 40/40] test: add failing test for addConstraint with schema --- test/integration/query-interface.test.js | 33 ++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/integration/query-interface.test.js b/test/integration/query-interface.test.js index 4f1a2a55c6fc..4f4f5c1ca74c 100644 --- a/test/integration/query-interface.test.js +++ b/test/integration/query-interface.test.js @@ -635,6 +635,39 @@ describe(Support.getTestDialectTeaser('QueryInterface'), () => { constraints = constraints.map(constraint => constraint.constraintName); expect(constraints).to.not.include(expectedConstraintName); }); + + // TODO: addConstraint does not support schemas yet. + it.skip('can add a constraint to a table in a non-default schema', async function () { + const tableName = { + tableName: 'users', + schema: 'archive', + }; + + await this.queryInterface.createTable(tableName, { + id: { + type: DataTypes.INTEGER, + }, + }); + + // changeColumn before addConstraint puts the DB2 table in "reorg pending state" + // addConstraint will be forced to execute a REORG TABLE command, which checks that it is done properly when using schemas. + await this.queryInterface.changeColumn(tableName, 'id', { + type: DataTypes.BIGINT, + }); + + await this.queryInterface.addConstraint(tableName, { + type: 'PRIMARY KEY', + fields: ['id'], + }); + + const constraints = await this.queryInterface.showConstraint(tableName); + + expect(constraints).to.deep.eq([{ + constraintName: 'users_username_pk', + schemaName: tableName.schema, + tableName: tableName.tableName, + }]); + }); }); describe('foreign key', () => {