Docker based test dbs (#3157)

* Fixed some of the tests failing with various dialect combinations * Setup all test databases to run in docker * Fixed test suite and updated travis to use docker * Fixed some tests in master and disabled oracle tests while setting everything else to work again * Changed docker to use latest postgresql alpine image * Re-enabled all oracledb tests * Install oracle client libs to oracle server and copy them to travis to make node oracledb package to work * Disabled random failing oracle tests and added test to show that oracle update in transaction does not seem to work ACID * Update package.json * Update package.json * Fixed linting issues * Fixed waiting for postgres to use the same DB image that database is using. * Removed installing oracledb driver in travis if not needed .travis.yml * Cleaned up package.json more * Created static name for oracledb container to allow copying files out of it on different docker versions * Removed webpack from deps
knex · May 13, 2019 · 8a9a648 · 8a9a648
1 parent cf435d9
commit 8a9a648
Show file tree

Hide file tree

Showing 21 changed files with 450 additions and 380 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -5,64 +5,43 @@ sudo: required
 
 cache:
   directories:
-  - "node_modules"
+    - 'node_modules'
 
 matrix:
   fast_finish: true
   include:
-  - node_js: "12"
-    env: DB="mssql mysql mysql2 postgres sqlite3" CXX=g++-4.8 KNEX_TEST_TIMEOUT=60000
-  - node_js: "10"
-    env: DB="mssql mysql mysql2 postgres sqlite3" CXX=g++-4.8 KNEX_TEST_TIMEOUT=60000
-  - node_js: "8"
-    env: DB="mssql mysql mysql2 postgres sqlite3" CXX=g++-4.8 KNEX_TEST_TIMEOUT=60000
-  - node_js: "6"
-    env: DB="mssql mysql mysql2 postgres sqlite3" CXX=g++-4.8 KNEX_TEST_TIMEOUT=60000
-  - node_js: "10"
-    env: TEST_ORACLEDB=true NODE_VER=11 KNEX_TEST_TIMEOUT=60000
-  - node_js: "10" # node_js env is just used to have npm command availabe, all test code is ran in docker container
-    env: TEST_ORACLEDB=true NODE_VER=10 KNEX_TEST_TIMEOUT=60000
-  - node_js: "10"
-    env: TEST_ORACLEDB=true NODE_VER=8 KNEX_TEST_TIMEOUT=60000
-  - node_js: "10"
-    env: TEST_ORACLEDB=true NODE_VER=6 KNEX_TEST_TIMEOUT=60000
-  allow_failures:
-  - node_js: "10"
-    env: TEST_ORACLEDB=true NODE_VER=11 KNEX_TEST_TIMEOUT=60000
-  - node_js: "10"
-    env: TEST_ORACLEDB=true NODE_VER=10 KNEX_TEST_TIMEOUT=60000
-  - node_js: "10"
-    env: TEST_ORACLEDB=true NODE_VER=8 KNEX_TEST_TIMEOUT=60000
-  - node_js: "10"
-    env: TEST_ORACLEDB=true NODE_VER=6 KNEX_TEST_TIMEOUT=60000
-
+    - node_js: '12'
+      env: DB="mssql mysql mysql2 postgres sqlite3" KNEX_TEST_TIMEOUT=60000
+    - node_js: '10'
+      env: DB="oracledb mssql mysql mysql2 postgres sqlite3" KNEX_TEST_TIMEOUT=60000
+    - node_js: '8'
+      env: DB="oracledb mssql mysql mysql2 postgres sqlite3" KNEX_TEST_TIMEOUT=60000
+    - node_js: '6'
+      env: DB="oracledb mssql mysql mysql2 postgres sqlite3" KNEX_TEST_TIMEOUT=60000
 install:
-  - if [ -z $TEST_ORACLEDB ]; then npm i; fi
+  - npm i
+  - (echo $DB | grep oracledb) && npm install oracledb || true
+  - npm run build
 
 before_script:
-  - if [ -z $TEST_ORACLEDB ]; then psql -c 'create database knex_test;' -U postgres; fi
-  - if [ -z $TEST_ORACLEDB ]; then mysql -e 'create database knex_test;'; fi
-  - if [ -z $TEST_ORACLEDB ]; then npm run mssql:init; fi
-  - if [ -z $TEST_ORACLEDB ]; then docker ps -a; fi
-  - if [ -z $TEST_ORACLEDB ]; then netstat -tulpn; fi
+  - npm run db:start
+  - echo "Install oracle client libs to db container and copy to hosts ~/lib for node oracledb driver..."
+  - docker-compose -f scripts/docker-compose.yml exec oracledbxe curl http://yum.oracle.com/public-yum-ol7.repo -o /etc/yum.repos.d/public-yum-ol7.repo
+  - docker-compose -f scripts/docker-compose.yml exec oracledbxe yum install -y yum-utils
+  - docker-compose -f scripts/docker-compose.yml exec oracledbxe yum-config-manager --enable ol7_oracle_instantclient
+  - docker-compose -f scripts/docker-compose.yml exec oracledbxe yum install -y oracle-instantclient18.3-basiclite
+  - docker cp oracledbxe_container:/usr/lib/oracle/18.3/client64/lib/ ~/
+  - sudo sh -c "echo $HOME/lib > /etc/ld.so.conf.d/oracle-instantclient.conf"
+  - sudo ldconfig
 
 script:
-  - if [ -z $TEST_ORACLEDB ]; then npm test; else npm run oracledb:test; fi
+  - npm test
 
 after_success:
-  - if [ -z $TEST_ORACLEDB ]; then npm run coveralls; fi
+  - npm run coveralls
 
 notifications:
   email: false
 
 services:
-  - mysql
-addons:
-  postgresql: '9.6'
-  apt:
-    packages:
-      - libmysqlclient-dev
-      - libmysqlclient20
-      - mysql-common
-      - mysql-server
-      - mysql-client
+  - docker
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -97,16 +97,37 @@ main();
 Usually issues without reproduction code available are just closed and if the same issue is reported multiple
 times maybe someone looks into it.
 
+One easy way to setup database for your reproduction is to use database from knex's docker-compose setup (npm run db:start) and by checking the connection settings from tests' `test/knexfile.js`.
+
 ## Integration Tests
 
 ### The Easy Way
 
-By default, Knex runs tests against MySQL (using [mysql](https://github.com/felixge/node-mysql) and [mysql2](https://github.com/sidorares/node-mysql2)), Postgres, and SQLite. The easiest way to run the tests is by creating the database `'knex_test'` and granting permissions to the database's default username:
+By default, Knex runs tests against sqlite3, postgresql, mysql, mysql2, mssql and oracledb drivers. All databases can be initialized and ran with docker.
+
+Docker databases can be started and initialized and started with:
+
+```bash
+npm run db:start
+```
+
+and stopped with:
+
+```bash
+npm run db:stop
+```
+
+### Installing support for oracledb
+
+Oracle has started providing precompiled driver libs for all the platforms, which makes it viable to run oracle tests also locally against oracledb running in docker.
 
-- **MySQL**: _root_
-- **Postgres**: _postgres_
+Check message when running
 
-No setup is required for SQLite.
+```bash
+npm install oracledb
+```
+
+and download driver library binary packages and unzip it to ~/lib directory.
 
 ### Specifying Databases
 
@@ -133,7 +154,9 @@ $ KNEX_TEST='./path/to/my/config.js' npm test
 
 ### Creating Postgres User
 
-Depending on your setup you might not have the default postgres user. To create a new user, login to Postgres and use the following queries to add the user. This assumes you've already created the `knex_test` database.
+If you are running tests agains own local database one might need to setup test user and databse for knex to connect.
+
+To create a new user, login to Postgres and use the following queries to add the user. This assumes you've already created the `knex_test` database.
 
 ```
 CREATE ROLE postgres WITH LOGIN PASSWORD '';
@@ -146,46 +169,6 @@ Once this is done, check it works by attempting to login:
 psql -h localhost -U postgres -d knex_test
 ```
 
-### Running OracleDB tests in docker
-
-Since node-oracledb driver is so hard to install on every platform, oracle tests
-are actually ran inside docker container. Container has Oracle XE g11,
-node 8 and node-oracledb driver installed and copies local knex directory in
-to be able to run the tests.
-
-```
-NODE_VER=10 npm run oracledb:test
-```
-
-You can also manually start shell in the docker image and run build commands manually:
-
-```
-docker run -i -t knex-test-oracledb /bin/bash
-
-root@34f1f1cd20cf:/#
-
-/usr/sbin/startup.sh
-cd knex
-npm install
-npm install oracledb
-npm test
-```
-
-### Runnin MSSQL SQL Server tests
-
-SQL Server needs to be started as docker container before running tests
-
-```
-# start mssql docker container
-npm run mssql:init
-
-# run tests, do changes etc.
-npm run mssql:test
-
-# stop mssql container
-npm run mssql:destroy
-```
-
 ## Want to be Collaborator?
 
 There is always room for more collaborators. Be active on resolving github issues / sending pull requests / reviewing code and we will ask you to join.

diff --git a/README.md b/README.md
@@ -28,6 +28,7 @@ If upgrading from older version, see [Upgrading instructions](https://github.com
 For support and questions, join the `#bookshelf` channel on freenode IRC
 
 For an Object Relational Mapper, see:
+
 - http://bookshelfjs.org
 - https://github.com/Vincit/objection.js
 
@@ -41,47 +42,51 @@ We have several examples [on the website](http://knexjs.org). Here is the first
 const knex = require('knex')({
   dialect: 'sqlite3',
   connection: {
-    filename: './data.db'
-  }
+    filename: './data.db',
+  },
 });
 
 // Create a table
-knex.schema.createTable('users', function(table) {
-  table.increments('id');
-  table.string('user_name');
-})
-
-// ...and another
-.createTable('accounts', function(table) {
-  table.increments('id');
-  table.string('account_name');
-  table.integer('user_id').unsigned().references('users.id');
-})
-
-// Then query the table...
-.then(function() {
-  return knex.insert({user_name: 'Tim'}).into('users');
-})
-
-// ...and using the insert id, insert into the other table.
-.then(function(rows) {
-  return knex.table('accounts').insert({account_name: 'knex', user_id: rows[0]});
-})
-
-// Query both of the rows.
-.then(function() {
-  return knex('users')
-    .join('accounts', 'users.id', 'accounts.user_id')
-    .select('users.user_name as user', 'accounts.account_name as account');
-})
-
-// .map over the results
-.map(function(row) {
-  console.log(row);
-})
-
-// Finally, add a .catch handler for the promise chain
-.catch(function(e) {
-  console.error(e);
-});
+knex.schema
+  .createTable('users', function(table) {
+    table.increments('id');
+    table.string('user_name');
+  })
+
+  // ...and another
+  .createTable('accounts', function(table) {
+    table.increments('id');
+    table.string('account_name');
+    table
+      .integer('user_id')
+      .unsigned()
+      .references('users.id');
+  })
+
+  // Then query the table...
+  .then(function() {
+    return knex('users').insert({ user_name: 'Tim' });
+  })
+
+  // ...and using the insert id, insert into the other table.
+  .then(function(rows) {
+    return knex('accounts').insert({ account_name: 'knex', user_id: rows[0] });
+  })
+
+  // Query both of the rows.
+  .then(function() {
+    return knex('users')
+      .join('accounts', 'users.id', 'accounts.user_id')
+      .select('users.user_name as user', 'accounts.account_name as account');
+  })
+
+  // .map over the results
+  .map(function(row) {
+    console.log(row);
+  })
+
+  // Finally, add a .catch handler for the promise chain
+  .catch(function(e) {
+    console.error(e);
+  });
 ```
diff --git a/package.json b/package.json
@@ -71,7 +71,8 @@
     "tape": "^4.10.1",
     "typescript": "^3.4.5",
     "through": "^2.3.8",
-    "toxiproxy-node-client": "^2.0.6"
+    "toxiproxy-node-client": "^2.0.6",
+    "webpack-cli": "^3.3.1"
   },
   "buildDependencies": [
     "@babel/cli",
@@ -80,29 +81,25 @@
     "rimraf"
   ],
   "scripts": {
-    "babel": "rimraf ./lib && babel src --out-dir lib --copy-files",
+    "format": "prettier --write \"{src,bin,scripts,test}/**/*.js\"",
     "build": "npm run babel",
-    "debug:test": "node --inspect-brk ./node_modules/.bin/_mocha -- --exit -t 0 test/index.js",
+    "babel": "rimraf ./lib && babel src --out-dir lib --copy-files",
+    "debug:test": "mocha --inspect-brk --exit -t 0 test/index.js",
     "debug:tape": "node --inspect-brk test/tape/index.js",
-    "format": "prettier --write \"{src,bin,scripts,test}/**/*.js\"",
     "coveralls": "nyc report --reporter=text-lcov | coveralls",
     "dev": "rimraf ./lib && babel -w src --out-dir lib --copy-files",
     "lint": "eslint '{src,test}/**/*.js'",
     "lint:types": "dtslint types",
-    "plaintest": "mocha --exit -t 10000 -b -R spec test/index.js && npm run tape",
+    "plaintest": "mocha --exit -t 10000 test/index.js && npm run test:tape && npm run test:cli",
     "plaintest:sqlite": "cross-env DB=sqlite3 npm run plaintest",
     "prepare": "npm run babel",
     "prepublishOnly": "npm run babel",
     "pretest": "npm run lint && npm run lint:types && npm run babel",
-    "test": "nyc mocha --exit --check-leaks --globals __core-js_shared__ -t 10000 -R spec test/index.js && npm run test:tape && npm run test:cli",
+    "test": "nyc mocha --exit --check-leaks --globals __core-js_shared__ -t 10000 test/index.js && npm run test:tape && npm run test:cli",
     "test:tape": "node test/tape/index.js | tap-spec",
     "test:cli": "cross-env KNEX_PATH=../knex.js KNEX=bin/cli.js jake -f test/jake/Jakefile",
-    "oracledb:test": "docker rmi -f --no-prune knex-test-oracledb && docker build -f scripts/oracle-tests-Dockerfile --tag knex-test-oracledb . && docker run --rm -i -t -e KNEX_TEST_TIMEOUT=$KNEX_TEST_TIMEOUT -e NODE_VER=$NODE_VER knex-test-oracledb",
-    "mssql:init": "docker-compose -f scripts/mssql-docker-compose.yml up --no-start && docker-compose -f scripts/mssql-docker-compose.yml start",
-    "mssql:logs": "docker-compose -f scripts/mssql-docker-compose.yml logs",
-    "mssql:test": "DB=mssql npm run plaintest",
-    "mssql:destroy": "docker-compose -f scripts/mssql-docker-compose.yml stop",
-    "postmssql:init": "node scripts/wait-for-mssql-server.js && npm run mssql:logs || (npm run mssql:logs;false)",
+    "db:start": "docker-compose -f scripts/docker-compose.yml up --build -d mysql oracledbxe postgres mssql; docker-compose -f scripts/docker-compose.yml up initmssqlknexdb waitmysql waitpostgres waitoracledbxe",
+    "db:stop": "docker-compose -f scripts/docker-compose.yml down",
     "stress:init": "docker-compose -f scripts/stress-test/docker-compose.yml up --no-start && docker-compose -f scripts/stress-test/docker-compose.yml start",
     "stress:test": "node scripts/stress-test/knex-stress-test.js |  grep -A 5 -B 60 -- '- STATS '",
     "stress:destroy": "docker-compose -f scripts/stress-test/docker-compose.yml stop"