diff --git a/lib/db/event-store.js b/lib/db/event-store.js index bee3771c934..a150ecdf84f 100644 --- a/lib/db/event-store.js +++ b/lib/db/event-store.js @@ -1,5 +1,6 @@ 'use strict'; +const { DROP_FEATURES } = require('../event-type'); const { EventEmitter } = require('events'); const EVENT_COLUMNS = ['id', 'type', 'created_by', 'created_at', 'data']; @@ -14,7 +15,7 @@ class EventStore extends EventEmitter { return this.db('events') .insert({ type: event.type, - created_by: event.createdBy, // eslint-disable-line + created_by: event.createdBy, // eslint-disable-line data: event.data, }) .then(() => this.emit(event.type, event)); @@ -35,6 +36,14 @@ class EventStore extends EventEmitter { .from('events') .limit(100) .whereRaw("data ->> 'name' = ?", [name]) + .andWhere( + 'id', + '>=', + this.db + .select(this.db.raw('coalesce(max(id),0) as id')) + .from('events') + .where({ type: DROP_FEATURES }) + ) .orderBy('created_at', 'desc') .map(this.rowToEvent); } diff --git a/lib/db/feature-toggle-store.js b/lib/db/feature-toggle-store.js index d61ee4083c8..932025ccf8d 100644 --- a/lib/db/feature-toggle-store.js +++ b/lib/db/feature-toggle-store.js @@ -5,6 +5,8 @@ const { FEATURE_UPDATED, FEATURE_ARCHIVED, FEATURE_REVIVED, + FEATURE_IMPORT, + DROP_FEATURES, } = require('../event-type'); const logger = require('../logger')('client-toggle-store.js'); const NotFoundError = require('../error/notfound-error'); @@ -33,6 +35,8 @@ class FeatureToggleStore { eventStore.on(FEATURE_REVIVED, event => this._reviveFeature(event.data) ); + eventStore.on(FEATURE_IMPORT, event => this._importFeature(event.data)); + eventStore.on(DROP_FEATURES, () => this._dropFeatures()); } getFeatures() { @@ -137,6 +141,29 @@ class FeatureToggleStore { logger.error('Could not archive feature, error was: ', err) ); } + + _importFeature(data) { + data = this.eventDataToRow(data); + return this.db + .raw(`? ON CONFLICT (name) DO ?`, [ + this.db(TABLE).insert(data), + this.db + .queryBuilder() + .update(data) + .update('archived', 0), + ]) + .catch(err => + logger.error('Could not import feature, error was: ', err) + ); + } + + _dropFeatures() { + return this.db(TABLE) + .delete() + .catch(err => + logger.error('Could not drop features, error was: ', err) + ); + } } module.exports = FeatureToggleStore; diff --git a/lib/db/strategy-store.js b/lib/db/strategy-store.js index 85642e9d664..575403cfaf1 100644 --- a/lib/db/strategy-store.js +++ b/lib/db/strategy-store.js @@ -4,6 +4,8 @@ const { STRATEGY_CREATED, STRATEGY_DELETED, STRATEGY_UPDATED, + STRATEGY_IMPORT, + DROP_STRATEGIES, } = require('../event-type'); const logger = require('../logger')('strategy-store.js'); const NotFoundError = require('../error/notfound-error'); @@ -19,14 +21,13 @@ class StrategyStore { eventStore.on(STRATEGY_UPDATED, event => this._updateStrategy(event.data) ); - eventStore.on(STRATEGY_DELETED, event => { - db(TABLE) - .where('name', event.data.name) - .del() - .catch(err => { - logger.error('Could not delete strategy, error was: ', err); - }); - }); + eventStore.on(STRATEGY_DELETED, event => + this._deleteStrategy(event.data) + ); + eventStore.on(STRATEGY_IMPORT, event => + this._importStrategy(event.data) + ); + eventStore.on(DROP_STRATEGIES, () => this._dropStrategies()); } getStrategies() { @@ -81,6 +82,36 @@ class StrategyStore { logger.error('Could not update strategy, error was: ', err) ); } + + _deleteStrategy({ name }) { + return this.db(TABLE) + .where({ name }) + .del() + .catch(err => { + logger.error('Could not delete strategy, error was: ', err); + }); + } + + _importStrategy(data) { + data = this.eventDataToRow(data); + return this.db + .raw(`? ON CONFLICT (name) DO ?`, [ + this.db(TABLE).insert(data), + this.db.queryBuilder().update(data).where(`${TABLE}.built_in`, 0), // eslint-disable-line + ]) + .catch(err => + logger.error('Could not import strategy, error was: ', err) + ); + } + + _dropStrategies() { + return this.db(TABLE) + .where({ built_in: 0 }) // eslint-disable-line + .delete() + .catch(err => + logger.error('Could not drop strategies, error was: ', err) + ); + } } module.exports = StrategyStore; diff --git a/lib/event-differ.js b/lib/event-differ.js index b29a849fecf..917c0141505 100644 --- a/lib/event-differ.js +++ b/lib/event-differ.js @@ -4,20 +4,32 @@ const { STRATEGY_CREATED, STRATEGY_DELETED, STRATEGY_UPDATED, + STRATEGY_IMPORT, + DROP_STRATEGIES, FEATURE_CREATED, FEATURE_UPDATED, FEATURE_ARCHIVED, FEATURE_REVIVED, + FEATURE_IMPORT, + DROP_FEATURES, } = require('./event-type'); const diff = require('deep-diff').diff; -const strategyTypes = [STRATEGY_CREATED, STRATEGY_DELETED, STRATEGY_UPDATED]; +const strategyTypes = [ + STRATEGY_CREATED, + STRATEGY_DELETED, + STRATEGY_UPDATED, + STRATEGY_IMPORT, + DROP_STRATEGIES, +]; const featureTypes = [ FEATURE_CREATED, FEATURE_UPDATED, FEATURE_ARCHIVED, FEATURE_REVIVED, + FEATURE_IMPORT, + DROP_FEATURES, ]; function baseTypeFor(event) { diff --git a/lib/event-type.js b/lib/event-type.js index 838d18c7462..95d2d6fe860 100644 --- a/lib/event-type.js +++ b/lib/event-type.js @@ -5,7 +5,11 @@ module.exports = { FEATURE_UPDATED: 'feature-updated', FEATURE_ARCHIVED: 'feature-archived', FEATURE_REVIVED: 'feature-revived', + FEATURE_IMPORT: 'feature-import', + DROP_FEATURES: 'drop-features', STRATEGY_CREATED: 'strategy-created', STRATEGY_DELETED: 'strategy-deleted', STRATEGY_UPDATED: 'strategy-updated', + STRATEGY_IMPORT: 'strategy-import', + DROP_STRATEGIES: 'drop-strategies', }; diff --git a/lib/options.js b/lib/options.js index bebaae61732..82157ea2671 100644 --- a/lib/options.js +++ b/lib/options.js @@ -20,6 +20,8 @@ const DEFAULT_OPTIONS = { sessionAge: THIRTY_DAYS, adminAuthentication: 'unsecure', ui: {}, + importFile: undefined, + dropBeforeImport: false, }; module.exports = { diff --git a/lib/routes/admin-api/api-def.json b/lib/routes/admin-api/api-def.json index 9be80b824aa..c0fbf518f0a 100644 --- a/lib/routes/admin-api/api-def.json +++ b/lib/routes/admin-api/api-def.json @@ -15,6 +15,9 @@ }, "metrics": { "uri": "/api/admin/metrics" + }, + "state": { + "uri": "/api/admin/state" } } } diff --git a/lib/routes/admin-api/index.js b/lib/routes/admin-api/index.js index 16de70e4d2c..af5261d1ac2 100644 --- a/lib/routes/admin-api/index.js +++ b/lib/routes/admin-api/index.js @@ -8,6 +8,7 @@ const StrategyController = require('./strategy'); const MetricsController = require('./metrics'); const UserController = require('./user'); const ConfigController = require('./config'); +const StateController = require('./state'); const apiDef = require('./api-def.json'); class AdminApi extends Controller { @@ -22,6 +23,7 @@ class AdminApi extends Controller { this.app.use('/metrics', new MetricsController(config).router); this.app.use('/user', new UserController(config).router); this.app.use('/ui-config', new ConfigController(config).router); + this.app.use('/state', new StateController(config).router); } index(req, res) { diff --git a/lib/routes/admin-api/state.js b/lib/routes/admin-api/state.js new file mode 100644 index 00000000000..e668ee39747 --- /dev/null +++ b/lib/routes/admin-api/state.js @@ -0,0 +1,80 @@ +'use strict'; + +const Controller = require('../controller'); +const { ADMIN } = require('../../permissions'); +const extractUser = require('../../extract-user'); +const { handleErrors } = require('./util'); +const YAML = require('js-yaml'); +const moment = require('moment'); +const multer = require('multer'); +const upload = multer({ limits: { fileSize: 5242880 } }); + +class ImportController extends Controller { + constructor(config) { + super(config); + this.fileupload('/import', upload.single('file'), this.import, ADMIN); + this.get('/export', this.export, ADMIN); + } + + async import(req, res) { + const userName = extractUser(req); + const { drop } = req.query; + + let data; + if (req.file) { + if (req.file.mimetype === 'text/yaml') { + data = YAML.safeLoad(req.file.buffer); + } else { + data = JSON.parse(req.file.buffer); + } + } else { + data = req.body; + } + + try { + await this.config.stateService.import({ + data, + userName, + dropBeforeImport: drop, + }); + res.sendStatus(202); + } catch (err) { + handleErrors(res, err); + } + } + + async export(req, res) { + const { format } = req.query; + + let strategies = 'strategies' in req.query; + let featureToggles = 'features' in req.query; + + if (!strategies && !featureToggles) { + strategies = true; + featureToggles = true; + } + + try { + const data = await this.config.stateService.export({ + strategies, + featureToggles, + }); + const timestamp = moment().format('YYYY-MM-DD_HH-mm-ss'); + if (format === 'yaml') { + if ('download' in req.query) { + res.attachment(`export-${timestamp}.yml`); + } + res.type('yaml').send(YAML.safeDump(data)); + } else { + if ('download' in req.query) { + res.attachment(`export-${timestamp}.json`); + } + res.json(data); + } + } catch (err) { + handleErrors(res, err); + } + } +} + +module.exports = ImportController; diff --git a/lib/routes/controller.js b/lib/routes/controller.js index 7909dc3f0bc..81d3330d94e 100644 --- a/lib/routes/controller.js +++ b/lib/routes/controller.js @@ -44,6 +44,15 @@ class Controller { ); } + fileupload(path, filehandler, handler, permission) { + this.app.post( + path, + checkPermission(this.config, permission), + filehandler, + handler.bind(this) + ); + } + use(path, router) { this.app.use(path, router); } diff --git a/lib/server-impl.js b/lib/server-impl.js index 31351a03917..a446f4562e8 100644 --- a/lib/server-impl.js +++ b/lib/server-impl.js @@ -10,10 +10,11 @@ const getApp = require('./app'); const { startMonitoring } = require('./metrics'); const { createStores } = require('./db'); const { createOptions } = require('./options'); +const StateService = require('./state-service'); const User = require('./user'); const AuthenticationRequired = require('./authentication-required'); -function createApp(options) { +async function createApp(options) { // Database dependecies (statefull) const stores = createStores(options); const eventBus = new EventEmitter(); @@ -35,12 +36,29 @@ function createApp(options) { stores.clientMetricsStore ); + const stateService = new StateService(config); + config.stateService = stateService; + if (config.importFile) { + await stateService.importFile({ + importFile: config.importFile, + dropBeforeImport: config.dropBeforeImport, + userName: 'importer', + }); + } + const server = app.listen({ port: options.port, host: options.host }, () => logger.info(`Unleash started on port ${server.address().port}`) ); - return new Promise((resolve, reject) => { - server.on('listening', () => resolve({ app, server, eventBus })); + return await new Promise((resolve, reject) => { + server.on('listening', () => + resolve({ + app, + server, + eventBus, + stateService, + }) + ); server.on('error', reject); }); } diff --git a/lib/state-service.js b/lib/state-service.js new file mode 100644 index 00000000000..0c9f7a60ee7 --- /dev/null +++ b/lib/state-service.js @@ -0,0 +1,122 @@ +'use strict'; + +const joi = require('joi'); +const fs = require('fs'); +const mime = require('mime'); +const { featureShema } = require('./routes/admin-api/feature-schema'); +const strategySchema = require('./routes/admin-api/strategy-schema'); +const getLogger = require('./logger'); +const yaml = require('js-yaml'); +const { + FEATURE_IMPORT, + DROP_FEATURES, + STRATEGY_IMPORT, + DROP_STRATEGIES, +} = require('./event-type'); + +const logger = getLogger('state-service.js'); + +const dataSchema = joi.object().keys({ + features: joi + .array() + .optional() + .items(featureShema), + strategies: joi + .array() + .optional() + .items(strategySchema), +}); + +class StateService { + constructor(config) { + this.config = config; + } + + async importFile({ importFile, dropBeforeImport, userName }) { + let data = await new Promise((resolve, reject) => + fs.readFile(importFile, (err, v) => + err ? reject(err) : resolve(v) + ) + ); + if (mime.lookup(importFile) === 'text/yaml') { + data = yaml.safeLoad(data); + } + + await this.import({ + data, + dropBeforeImport, + userName, + }); + } + + async import({ data, userName, dropBeforeImport }) { + const { eventStore } = this.config.stores; + + if (typeof data === 'string') { + data = JSON.parse(data); + } + + data = await joi.validate(data, dataSchema); + + if (data.features) { + logger.info(`Importing ${data.features.length} features`); + if (dropBeforeImport) { + logger.info(`Dropping existing features`); + await eventStore.store({ + type: DROP_FEATURES, + createdBy: userName, + data: { name: 'all-features' }, + }); + } + for (const feature of data.features) { + await eventStore.store({ + type: FEATURE_IMPORT, + createdBy: userName, + data: feature, + }); + } + } + + if (data.strategies) { + logger.info(`Importing ${data.strategies.length} strategies`); + if (dropBeforeImport) { + logger.info(`Dropping existing strategies`); + await eventStore.store({ + type: DROP_STRATEGIES, + createdBy: userName, + data: { name: 'all-strategies' }, + }); + } + for (const strategy of data.strategies) { + await eventStore.store({ + type: STRATEGY_IMPORT, + createdBy: userName, + data: strategy, + }); + } + } + } + + async export({ strategies, featureToggles }) { + const { featureToggleStore, strategyStore } = this.config.stores; + const result = {}; + + if (featureToggles) { + result.features = await featureToggleStore.getFeatures(); + } + + if (strategies) { + result.strategies = (await strategyStore.getStrategies()) + .filter(strat => strat.editable) + .map(strat => { + strat = Object.assign({}, strat); + delete strat.editable; + return strat; + }); + } + + return result; + } +} + +module.exports = StateService; diff --git a/lib/state-service.test.js b/lib/state-service.test.js new file mode 100644 index 00000000000..3f1c04e1591 --- /dev/null +++ b/lib/state-service.test.js @@ -0,0 +1,148 @@ +'use strict'; + +const test = require('ava'); + +const store = require('./../test/fixtures/store'); +const StateService = require('./state-service'); +const { + FEATURE_IMPORT, + DROP_FEATURES, + STRATEGY_IMPORT, + DROP_STRATEGIES, +} = require('./event-type'); + +function getSetup() { + const stores = store.createStores(); + return { stateService: new StateService({ stores }), stores }; +} + +test('should import a feature', async t => { + const { stateService, stores } = getSetup(); + + const data = { + features: [ + { + name: 'new-feature', + enabled: true, + strategies: [{ name: 'default' }], + }, + ], + }; + + await stateService.import({ data }); + + const events = await stores.eventStore.getEvents(); + t.is(events.length, 1); + t.is(events[0].type, FEATURE_IMPORT); + t.is(events[0].data.name, 'new-feature'); +}); + +test('should drop feature before import if specified', async t => { + const { stateService, stores } = getSetup(); + + const data = { + features: [ + { + name: 'new-feature', + enabled: true, + strategies: [{ name: 'default' }], + }, + ], + }; + + await stateService.import({ data, dropBeforeImport: true }); + + const events = await stores.eventStore.getEvents(); + t.is(events.length, 2); + t.is(events[0].type, DROP_FEATURES); + t.is(events[1].type, FEATURE_IMPORT); + t.is(events[1].data.name, 'new-feature'); +}); + +test('should import a strategy', async t => { + const { stateService, stores } = getSetup(); + + const data = { + strategies: [ + { + name: 'new-strategy', + parameters: [], + }, + ], + }; + + await stateService.import({ data }); + + const events = await stores.eventStore.getEvents(); + t.is(events.length, 1); + t.is(events[0].type, STRATEGY_IMPORT); + t.is(events[0].data.name, 'new-strategy'); +}); + +test('should drop strategies before import if specified', async t => { + const { stateService, stores } = getSetup(); + + const data = { + strategies: [ + { + name: 'new-strategy', + parameters: [], + }, + ], + }; + + await stateService.import({ data, dropBeforeImport: true }); + + const events = await stores.eventStore.getEvents(); + t.is(events.length, 2); + t.is(events[0].type, DROP_STRATEGIES); + t.is(events[1].type, STRATEGY_IMPORT); + t.is(events[1].data.name, 'new-strategy'); +}); + +test('should drop neither features nor strategies when neither is imported', async t => { + const { stateService, stores } = getSetup(); + + const data = {}; + + await stateService.import({ data, dropBeforeImport: true }); + + const events = await stores.eventStore.getEvents(); + t.is(events.length, 0); +}); + +test('should not accept gibberish', async t => { + const { stateService } = getSetup(); + + const data1 = { + type: 'gibberish', + flags: { evil: true }, + }; + const data2 = '{somerandomtext/'; + + await t.throwsAsync(stateService.import({ data: data1 })); + + await t.throwsAsync(stateService.import({ data: data2 })); +}); + +test('should export featureToggles', async t => { + const { stateService, stores } = getSetup(); + + stores.featureToggleStore.addFeature({ name: 'a-feature' }); + + const data = await stateService.export({ featureToggles: true }); + + t.is(data.features.length, 1); + t.is(data.features[0].name, 'a-feature'); +}); + +test('should export strategies', async t => { + const { stateService, stores } = getSetup(); + + stores.strategyStore.addStrategy({ name: 'a-strategy', editable: true }); + + const data = await stateService.export({ strategies: true }); + + t.is(data.strategies.length, 1); + t.is(data.strategies[0].name, 'a-strategy'); +}); diff --git a/package.json b/package.json index e9f74485e5c..4c9c81e7ab2 100644 --- a/package.json +++ b/package.json @@ -69,9 +69,12 @@ "gravatar": "^1.8.0", "install": "^0.12.2", "joi": "^14.3.1", + "js-yaml": "^3.12.2", "knex": "^0.16.3", "log4js": "^4.0.0", + "mime": "^1.4.1", "moment": "^2.24.0", + "multer": "^1.4.1", "parse-database-url": "^0.3.0", "pg": "^7.8.1", "pkginfo": "^0.4.1", diff --git a/test/fixtures/fake-event-store.js b/test/fixtures/fake-event-store.js index ee1fb29d885..ff961bf9944 100644 --- a/test/fixtures/fake-event-store.js +++ b/test/fixtures/fake-event-store.js @@ -6,7 +6,7 @@ module.exports = () => { return { store: event => { events.push(event); - Promise.resolve(); + return Promise.resolve(); }, getEvents: () => Promise.resolve(events), };