From c09c20ad8d780570a28757641000f6a76d32eb38 Mon Sep 17 00:00:00 2001 From: Sebastian Gierlinger Date: Thu, 12 Jun 2014 17:25:55 +0200 Subject: [PATCH] Allow schema changes closes #2354 refs #1641 - added addUnique() - added dropUnique() - added addColumn() -> needed for #2330 - dropColumn() is missing due to lack of knex support - further cleanup of the migrations module --- core/server/data/migration/index.js | 84 +++++++++++++-- core/server/data/schema.js | 4 +- core/server/data/utils/index.js | 154 +++++++++++++++++----------- core/server/data/utils/mysql.js | 28 +++++ core/server/data/utils/pgsql.js | 33 ++++++ core/server/data/utils/sqlite3.js | 29 ++++++ 6 files changed, 260 insertions(+), 72 deletions(-) create mode 100644 core/server/data/utils/mysql.js create mode 100644 core/server/data/utils/pgsql.js create mode 100644 core/server/data/utils/sqlite3.js diff --git a/core/server/data/migration/index.js b/core/server/data/migration/index.js index 92e4b4f9da3c..d9c60d681bc0 100644 --- a/core/server/data/migration/index.js +++ b/core/server/data/migration/index.js @@ -44,6 +44,36 @@ function getAddCommands(oldTables, newTables) { } } +function addColumnCommands(table, columns) { + var columnKeys = _.keys(schema[table]), + addColumns = _.difference(columnKeys, columns); + + return _.map(addColumns, function (column) { + return function () { + utils.addColumn(table, column); + }; + }); +} + +function modifyUniqueCommands(table, indexes) { + var columnKeys = _.keys(schema[table]); + return _.map(columnKeys, function (column) { + if (schema[table][column].unique && schema[table][column].unique === true) { + if (!_.contains(indexes, table + '_' + column + '_unique')) { + return function () { + return utils.addUnique(table, column); + }; + } + } else if (!schema[table][column].unique) { + if (_.contains(indexes, table + '_' + column + '_unique')) { + return function () { + return utils.dropUnique(table, column); + }; + } + } + }); +} + // Check for whether data is needed to be bootstrapped or not init = function () { var self = this; @@ -144,27 +174,61 @@ function backupDatabase() { // Migrate from a specific version to the latest migrateUp = function () { + var deleteCommands, + addCommands, + oldTables, + addColumns = [], + modifyUniCommands = [], + commands = []; + return backupDatabase().then(function () { - return utils.getTables(); - }).then(function (oldTables) { + return utils.getTables().then(function (tables) { + oldTables = tables; + }); + }).then(function () { // if tables exist and client is mysqls check if posts table is okay if (!_.isEmpty(oldTables) && client === 'mysql') { - return checkMySQLPostTable().then(function () { - return oldTables; - }); + return checkMySQLPostTable(); } - return oldTables; - }).then(function (oldTables) { - var deleteCommands = getDeleteCommands(oldTables, schemaTables), - addCommands = getAddCommands(oldTables, schemaTables), - commands = []; + }).then(function () { + deleteCommands = getDeleteCommands(oldTables, schemaTables); + addCommands = getAddCommands(oldTables, schemaTables); + return when.all( + _.map(oldTables, function (table) { + return utils.getIndexes(table).then(function (indexes) { + modifyUniCommands = modifyUniCommands.concat(modifyUniqueCommands(table, indexes)); + }); + }) + ); + }).then(function () { + return when.all( + _.map(oldTables, function (table) { + return utils.getColumns(table).then(function (columns) { + addColumns = addColumns.concat(addColumnCommands(table, columns)); + }); + }) + ); + + }).then(function () { + modifyUniCommands = _.compact(modifyUniCommands); + // delete tables if (!_.isEmpty(deleteCommands)) { commands = commands.concat(deleteCommands); } + // add tables if (!_.isEmpty(addCommands)) { commands = commands.concat(addCommands); } + // add columns if needed + if (!_.isEmpty(addColumns)) { + commands = commands.concat(addColumns); + } + // add/drop unique constraint + if (!_.isEmpty(modifyUniCommands)) { + commands = commands.concat(modifyUniCommands); + } + // execute the commands in sequence if (!_.isEmpty(commands)) { return sequence(commands); } diff --git a/core/server/data/schema.js b/core/server/data/schema.js index bdbaf9d41277..d2ecd03a870c 100644 --- a/core/server/data/schema.js +++ b/core/server/data/schema.js @@ -24,8 +24,8 @@ var db = { users: { id: {type: 'increments', nullable: false, primary: true}, uuid: {type: 'string', maxlength: 36, nullable: false, validations: {'isUUID': true}}, - name: {type: 'string', maxlength: 150, nullable: false, unique: true}, - slug: {type: 'string', maxlength: 150, nullable: false}, + name: {type: 'string', maxlength: 150, nullable: false}, + slug: {type: 'string', maxlength: 150, nullable: false, unique: true}, password: {type: 'string', maxlength: 60, nullable: false}, email: {type: 'string', maxlength: 254, nullable: false, unique: true, validations: {'isEmail': true}}, image: {type: 'text', maxlength: 2000, nullable: true}, diff --git a/core/server/data/utils/index.js b/core/server/data/utils/index.js index 08bb17169cb7..865022322b3d 100644 --- a/core/server/data/utils/index.js +++ b/core/server/data/utils/index.js @@ -1,83 +1,112 @@ -var _ = require('lodash'), - when = require('when'), - knex = require('../../models/base').knex, - schema = require('../schema').tables, - client = require('../../models/base').client; +var _ = require('lodash'), + when = require('when'), + knex = require('../../models/base').knex, + schema = require('../schema').tables, + client = require('../../models/base').client, + sqlite3 = require('./sqlite3'), + mysql = require('./mysql'), + pgsql = require('./pgsql'); -function createTable(table) { - return knex.schema.createTable(table, function (t) { - var column, - columnKeys = _.keys(schema[table]); - _.each(columnKeys, function (key) { - // creation distinguishes between text with fieldtype, string with maxlength and all others - if (schema[table][key].type === 'text' && schema[table][key].hasOwnProperty('fieldtype')) { - column = t[schema[table][key].type](key, schema[table][key].fieldtype); - } else if (schema[table][key].type === 'string' && schema[table][key].hasOwnProperty('maxlength')) { - column = t[schema[table][key].type](key, schema[table][key].maxlength); - } else { - column = t[schema[table][key].type](key); - } +function addTableColumn(tablename, table, columnname) { + var column; + // creation distinguishes between text with fieldtype, string with maxlength and all others + if (schema[tablename][columnname].type === 'text' && schema[tablename][columnname].hasOwnProperty('fieldtype')) { + column = table[schema[tablename][columnname].type](columnname, schema[tablename][columnname].fieldtype); + } else if (schema[tablename][columnname].type === 'string' && schema[tablename][columnname].hasOwnProperty('maxlength')) { + column = table[schema[tablename][columnname].type](columnname, schema[tablename][columnname].maxlength); + } else { + column = table[schema[tablename][columnname].type](columnname); + } - if (schema[table][key].hasOwnProperty('nullable') && schema[table][key].nullable === true) { - column.nullable(); - } else { - column.notNullable(); - } - if (schema[table][key].hasOwnProperty('primary') && schema[table][key].primary === true) { - column.primary(); - } - if (schema[table][key].hasOwnProperty('unique') && schema[table][key].unique) { - column.unique(); - } - if (schema[table][key].hasOwnProperty('unsigned') && schema[table][key].unsigned) { - column.unsigned(); - } - if (schema[table][key].hasOwnProperty('references')) { - //check if table exists? - column.references(schema[table][key].references); - } - if (schema[table][key].hasOwnProperty('defaultTo')) { - column.defaultTo(schema[table][key].defaultTo); - } - }); - }); + if (schema[tablename][columnname].hasOwnProperty('nullable') && schema[tablename][columnname].nullable === true) { + column.nullable(); + } else { + column.notNullable(); + } + if (schema[tablename][columnname].hasOwnProperty('primary') && schema[tablename][columnname].primary === true) { + column.primary(); + } + if (schema[tablename][columnname].hasOwnProperty('unique') && schema[tablename][columnname].unique) { + column.unique(); + } + if (schema[tablename][columnname].hasOwnProperty('unsigned') && schema[tablename][columnname].unsigned) { + column.unsigned(); + } + if (schema[tablename][columnname].hasOwnProperty('references')) { + //check if table exists? + column.references(schema[tablename][columnname].references); + } + if (schema[tablename][columnname].hasOwnProperty('defaultTo')) { + column.defaultTo(schema[tablename][columnname].defaultTo); + } } -function deleteTable(table) { - return knex.schema.dropTableIfExists(table); +function addColumn(table, column) { + return knex.schema.table(table, function (t) { + addTableColumn(table, t, column); + }); } -function getTablesFromSqlite3() { - return knex.raw("select * from sqlite_master where type = 'table'").then(function (response) { - return _.reject(_.pluck(response, 'tbl_name'), function (name) { - return name === 'sqlite_sequence'; - }); +function addUnique(table, column) { + return knex.schema.table(table, function (table) { + table.unique(column); }); } -function getTablesFromPgSQL() { - return knex.raw("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'").then(function (response) { - return _.flatten(_.pluck(response.rows, 'table_name')); +function dropUnique(table, column) { + return knex.schema.table(table, function (table) { + table.dropUnique(column); }); } -function getTablesFromMySQL() { - return knex.raw('show tables').then(function (response) { - return _.flatten(_.map(response[0], function (entry) { - return _.values(entry); - })); +function createTable(table) { + return knex.schema.createTable(table, function (t) { + var columnKeys = _.keys(schema[table]); + _.each(columnKeys, function (column) { + return addTableColumn(table, t, column); + }); }); } +function deleteTable(table) { + return knex.schema.dropTableIfExists(table); +} + function getTables() { if (client === 'sqlite3') { - return getTablesFromSqlite3(); + return sqlite3.getTables(); + } + if (client === 'mysql') { + return mysql.getTables(); + } + if (client === 'pg') { + return pgsql.getTables(); + } + return when.reject("No support for database client " + client); +} + +function getIndexes(table) { + if (client === 'sqlite3') { + return sqlite3.getIndexes(table); + } + if (client === 'mysql') { + return mysql.getIndexes(table); + } + if (client === 'pg') { + return pgsql.getIndexes(table); + } + return when.reject("No support for database client " + client); +} + +function getColumns(table) { + if (client === 'sqlite3') { + return sqlite3.getColumns(table); } if (client === 'mysql') { - return getTablesFromMySQL(); + return mysql.getColumns(table); } if (client === 'pg') { - return getTablesFromPgSQL(); + return pgsql.getColumns(table); } return when.reject("No support for database client " + client); } @@ -85,5 +114,10 @@ function getTables() { module.exports = { createTable: createTable, deleteTable: deleteTable, - getTables: getTables + getTables: getTables, + getIndexes: getIndexes, + addUnique: addUnique, + dropUnique: dropUnique, + addColumn: addColumn, + getColumns: getColumns }; diff --git a/core/server/data/utils/mysql.js b/core/server/data/utils/mysql.js new file mode 100644 index 000000000000..7b3bbbd9d909 --- /dev/null +++ b/core/server/data/utils/mysql.js @@ -0,0 +1,28 @@ +var _ = require('lodash'), + knex = require('../../models/base').knex; + +function getTables() { + return knex.raw('show tables').then(function (response) { + return _.flatten(_.map(response[0], function (entry) { + return _.values(entry); + })); + }); +} + +function getIndexes(table) { + return knex.raw('SHOW INDEXES from ' + table).then(function (response) { + return _.flatten(_.pluck(response[0], 'Key_name')); + }); +} + +function getColumns(table) { + return knex.raw('SHOW COLUMNS FROM ' + table).then(function (response) { + return _.flatten(_.pluck(response[0], 'Field')); + }); +} + +module.exports = { + getTables: getTables, + getIndexes: getIndexes, + getColumns: getColumns +}; \ No newline at end of file diff --git a/core/server/data/utils/pgsql.js b/core/server/data/utils/pgsql.js new file mode 100644 index 000000000000..39683e139328 --- /dev/null +++ b/core/server/data/utils/pgsql.js @@ -0,0 +1,33 @@ +var _ = require('lodash'), + knex = require('../../models/base').knex; + +function getTables() { + return knex.raw("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'").then(function (response) { + return _.flatten(_.pluck(response.rows, 'table_name')); + }); +} + +function getIndexes(table) { + var selectIndexes = "SELECT t.relname as table_name, i.relname as index_name, a.attname as column_name" + + " FROM pg_class t, pg_class i, pg_index ix, pg_attribute a" + + " WHERE t.oid = ix.indrelid and i.oid = ix.indexrelid and" + + " a.attrelid = t.oid and a.attnum = ANY(ix.indkey) and t.relname = '" + table + "'"; + + return knex.raw(selectIndexes).then(function (response) { + return _.flatten(_.pluck(response.rows, 'index_name')); + }); +} + +function getColumns(table) { + var selectIndexes = "SELECT column_name FROM information_schema.columns WHERE table_name = '" + table + "'"; + + return knex.raw(selectIndexes).then(function (response) { + return _.flatten(_.pluck(response.rows, 'column_name')); + }); +} + +module.exports = { + getTables: getTables, + getIndexes: getIndexes, + getColumns: getColumns +}; \ No newline at end of file diff --git a/core/server/data/utils/sqlite3.js b/core/server/data/utils/sqlite3.js new file mode 100644 index 000000000000..45c779c62ebe --- /dev/null +++ b/core/server/data/utils/sqlite3.js @@ -0,0 +1,29 @@ +var _ = require('lodash'), + knex = require('../../models/base').knex; + +function getTables() { + return knex.raw("select * from sqlite_master where type = 'table'").then(function (response) { + return _.reject(_.pluck(response, 'tbl_name'), function (name) { + return name === 'sqlite_sequence'; + }); + }); +} + +function getIndexes(table) { + return knex.raw("pragma index_list('" + table + "')").then(function (response) { + + return _.flatten(_.pluck(response, 'name')); + }); +} + +function getColumns(table) { + return knex.raw("pragma table_info('" + table + "')").then(function (response) { + return _.flatten(_.pluck(response, 'name')); + }); +} + +module.exports = { + getTables: getTables, + getIndexes: getIndexes, + getColumns: getColumns +}; \ No newline at end of file