HA: multiple instance replica support

forge/db/models/Application.js

@@ -2,9 +2,9 @@
  * An application definition
  * @namespace forge.db.models.Application
  */
-const { DataTypes } = require('sequelize')
+const { DataTypes, Op } = require('sequelize')
 
-const { KEY_SETTINGS } = require('./ProjectSettings')
+const { KEY_SETTINGS, KEY_HA } = require('./ProjectSettings')
 
 module.exports = {
     name: 'Application',
@@ -60,7 +60,12 @@ module.exports = {
                                     attributes: ['hashid', 'id', 'name', 'links', 'settings', 'policy']
                                 }, {
                                     model: M.ProjectSettings,
-                                    where: { key: KEY_SETTINGS },
+                                    where: {
+                                        [Op.or]: [
+                                            { key: KEY_SETTINGS },
+                                            { key: KEY_HA }
+                                        ]
+                                    },
                                     required: false
                                 }
                             ]

forge/db/models/Project.js

@@ -14,7 +14,7 @@
 const { DataTypes, Op } = require('sequelize')
 const Controllers = require('../controllers')
 
-const { KEY_HOSTNAME, KEY_SETTINGS } = require('./ProjectSettings')
+const { KEY_HOSTNAME, KEY_SETTINGS, KEY_HA } = require('./ProjectSettings')
 
 /** @type {FFModel} */
 module.exports = {
@@ -207,7 +207,7 @@ module.exports = {
                     if (key === 'settings' && value && Array.isArray(value.env)) {
                         value.env = Controllers.Project.removePlatformSpecificEnvVars(value.env) // remove platform specific values
                     }
-                    return await M.ProjectSettings.upsert({ ProjectId: this.id, key, value }, options)
+                    return M.ProjectSettings.upsert({ ProjectId: this.id, key, value }, options)
                 },
                 async getSetting (key) {
                     const result = await M.ProjectSettings.findOne({ where: { ProjectId: this.id, key } })
@@ -220,7 +220,9 @@ module.exports = {
                     }
                     return undefined
                 },
-
+                async removeSetting (key, options) {
+                    return M.ProjectSettings.destroy({ where: { ProjectId: this.id, key } }, options)
+                },
                 async getCredentialSecret () {
                     // If this project was created at 0.6+ but then started with a <0.6 launcher
                     // (for example, in k8s with an old stack) then the project will have both
@@ -335,7 +337,8 @@ module.exports = {
                                 where: {
                                     [Op.or]: [
                                         { key: KEY_SETTINGS },
-                                        { key: KEY_HOSTNAME }
+                                        { key: KEY_HOSTNAME },
+                                        { key: KEY_HA }
                                     ]
                                 },
                                 required: false
@@ -371,7 +374,12 @@ module.exports = {
 
                         include.push({
                             model: M.ProjectSettings,
-                            where: { key: KEY_SETTINGS },
+                            where: {
+                                [Op.or]: [
+                                    { key: KEY_SETTINGS },
+                                    { key: KEY_HA }
+                                ]
+                            },
                             required: false
                         })
                     }

forge/db/models/ProjectSettings.js

@@ -12,6 +12,7 @@ const SettingTypes = {
 const KEY_SETTINGS = 'settings'
 const KEY_HOSTNAME = 'hostname'
 const KEY_BILLING_STATE = 'billingState'
+const KEY_HA = 'ha'
 
 const BILLING_STATES = {
     UNKNOWN: undefined,
@@ -25,6 +26,7 @@ module.exports = {
     KEY_SETTINGS,
     KEY_HOSTNAME,
     KEY_BILLING_STATE,
+    KEY_HA,
     name: 'ProjectSettings',
     schema: {
         ProjectId: { type: DataTypes.UUID, unique: 'pk_settings' },

forge/db/views/Project.js

@@ -1,4 +1,4 @@
-const { KEY_HOSTNAME, KEY_SETTINGS } = require('../models/ProjectSettings')
+const { KEY_HOSTNAME, KEY_SETTINGS, KEY_HA } = require('../models/ProjectSettings')
 
 module.exports = {
     project: async function (app, project, { includeSettings = true } = {}) {
@@ -35,6 +35,10 @@ module.exports = {
             const settingsHostnameRow = proj.ProjectSettings?.find((projectSettingsRow) => projectSettingsRow.key === KEY_HOSTNAME)
             result.hostname = settingsHostnameRow?.value || ''
         }
+        if (app.config.features.enabled('ha')) {
+            const settingsHARow = proj.ProjectSettings?.find(row => row.key === KEY_HA)
+            result.ha = settingsHARow?.value || { disabled: true }
+        }
 
         if (proj.Application) {
             result.application = app.db.views.Application.applicationSummary(proj.Application)

forge/ee/lib/ha/index.js

@@ -0,0 +1,41 @@
+const { KEY_HA } = require('../../../db/models/ProjectSettings')
+
+module.exports.init = function (app) {
+    if (app.config.driver.type === 'k8s' || app.config.driver.type === 'stub') {
+        // Register ha flag as a private flag - no requirement to expose it in public settings
+        app.config.features.register('ha', true)
+
+        /**
+         * Check if HA is allowed for this given team/projectType/haConfig combination
+         * @param {*} team
+         * @param {*} projectType
+         * @param {*} haConfig
+         * @returns true/false
+         */
+        async function isHAAllowed (team, projectType, haConfig) {
+            // For initial beta release, we will support 1-2 replicas.
+            // 1 replica is equivalent to no HA
+            // In the future this will need to take into account the team type
+            return (haConfig.replicas > 0 && haConfig.replicas < 3)
+        }
+
+        // Add ha functions to the Project model
+        app.db.models.Project.prototype.getHASettings = async function () {
+            return this.getSetting(KEY_HA)
+        }
+        app.db.models.Project.prototype.updateHASettings = async function (haConfig) {
+            if (!haConfig) {
+                return this.removeSetting(KEY_HA)
+            }
+            if (haConfig?.replicas > 0 && haConfig?.replicas < 3) {
+                return this.updateSetting(KEY_HA, {
+                    replicas: haConfig.replicas
+                })
+            }
+        }
+
+        app.decorate('ha', {
+            isHAAllowed
+        })
+    }
+}

forge/ee/lib/index.js

@@ -6,6 +6,7 @@ module.exports = fp(async function (app, opts, done) {
     }
     require('./projectComms').init(app)
     require('./deviceEditor').init(app)
+    require('./ha').init(app)
 
     app.decorate('sso', await require('./sso').init(app))
 

forge/ee/routes/ha/index.js

@@ -0,0 +1,107 @@
+module.exports = async function (app) {
+    app.addHook('preHandler', app.verifySession)
+    app.addHook('preHandler', async (request, reply) => {
+        if (!app.config.features.enabled('ha')) {
+            reply.code(404).send({ code: 'not_found', error: 'Not Found' })
+        }
+    })
+    app.addHook('preHandler', async (request, reply) => {
+        if (request.params.projectId !== undefined) {
+            if (request.params.projectId) {
+                try {
+                    request.project = await app.db.models.Project.byId(request.params.projectId)
+                    if (!request.project) {
+                        reply.code(404).send({ code: 'not_found', error: 'Not Found' })
+                        return
+                    }
+                    if (request.session.User) {
+                        request.teamMembership = await request.session.User.getTeamMembership(request.project.Team.id)
+                        if (!request.teamMembership && !request.session.User.admin) {
+                            reply.code(404).send({ code: 'not_found', error: 'Not Found' })
+                            return
+                        }
+                    } else if (request.session.ownerId !== request.params.projectId) {
+                        // AccesToken being used - but not owned by this project
+                        reply.code(404).send({ code: 'not_found', error: 'Not Found' })
+                        return
+                    }
+                } catch (err) {
+                    reply.code(404).send({ code: 'not_found', error: 'Not Found' })
+                }
+            } else {
+                reply.code(404).send({ code: 'not_found', error: 'Not Found' })
+            }
+        }
+    })
+
+    app.get('/', {
+        preHandler: app.needsPermission('project:read')
+    }, async (request, reply) => {
+        reply.send(await request.project.getHASettings() || {})
+    })
+
+    app.put('/', {
+        preHandler: app.needsPermission('project:edit')
+    }, async (request, reply) => {
+        // For 1.8, only support setting replica count to 2
+        if (request.body.replicas !== 2) {
+            reply.code(409).send({ code: 'invalid_ha_configuration', error: 'Invalid HA configuration -only 2 replicas are allowed' })
+            return
+        }
+        const existingHA = await request.project.getHASettings() || {}
+        if (existingHA.replicas !== request.body.replicas) {
+            // This is a change in replica count.
+            await request.project.updateHASettings({ replicas: request.body.replicas })
+            await applyUpdatedInstanceSettings(reply, request.project, request.session.User)
+        } else {
+            reply.send(await request.project.getHASettings() || {})
+        }
+    })
+
+    app.delete('/', {
+        preHandler: app.needsPermission('project:edit')
+    }, async (request, reply) => {
+        const existingHA = await request.project.getHASettings() || {}
+        if (existingHA.replicas) {
+            // This instance already has ha configured - clear the setting
+            // and apply
+            await request.project.updateHASettings(undefined)
+            await applyUpdatedInstanceSettings(reply, request.project, request.session.User)
+        } else {
+            reply.send({})
+        }
+    })
+
+    async function applyUpdatedInstanceSettings (reply, project, user) {
+        if (project.state !== 'suspended') {
+            // This code is copy/paste with slight changes from projects.js
+            // We also have projectActions.js that does suspend logic.
+            // TODO: refactor into a Model function to suspend a project
+            app.db.controllers.Project.setInflightState(project, 'starting') // TODO: better inflight state needed
+            reply.send(await project.getHASettings() || {})
+
+            const targetState = project.state
+            app.log.info(`Stopping project ${project.id}`)
+            await app.containers.stop(project, {
+                skipBilling: true
+            })
+            await app.auditLog.Project.project.suspended(user, null, project)
+            app.log.info(`Restarting project ${project.id}`)
+            project.state = targetState
+            await project.save()
+            await project.reload()
+            const startResult = await app.containers.start(project)
+            startResult.started.then(async () => {
+                await app.auditLog.Project.project.started(user, null, project)
+                app.db.controllers.Project.clearInflightState(project)
+                return true
+            }).catch(_ => {
+                app.db.controllers.Project.clearInflightState(project)
+            })
+        } else {
+            // A suspended project doesn't need to do anything more
+            // The settings will get applied when it is next resumed
+            reply.send(await project.getHASettings() || {})
+        }
+    }
+}

forge/ee/routes/index.js

@@ -13,6 +13,8 @@ module.exports = async function (app) {
     await app.register(require('./pipeline'), { prefix: '/api/v1', logLevel: app.config.logging.http })
     await app.register(require('./deviceEditor'), { prefix: '/api/v1/devices/:deviceId/editor', logLevel: app.config.logging.http })
 
+    await app.register(require('./ha'), { prefix: '/api/v1/projects/:projectId/ha', logLevel: app.config.logging.http })
+
     // Important: keep SSO last to avoid its error handling polluting other routes.
     await app.register(require('./sso'), { logLevel: app.config.logging.http })
 }

forge/routes/api/project.js

@@ -193,6 +193,13 @@ module.exports = async function (app) {
             return
         }
 
+        if (app.license.active() && app.ha) {
+            if (request.body.ha && !await app.ha.isHAAllowed(team, projectType, request.body.ha)) {
+                reply.code(400).send({ code: 'invalid_ha', error: 'Invalid HA configuration' })
+                return
+            }
+        }
+
         let project
         try {
             project = await app.db.models.Project.create({
@@ -212,15 +219,20 @@ module.exports = async function (app) {
         await project.setProjectStack(stack)
         await project.setProjectTemplate(template)
         await project.setProjectType(projectType)
+
+        if (app.license.active() && app.ha && request.body.ha) {
+            await project.updateHASettings(request.body.ha)
+        }
+
         await project.reload({
             include: [
                 { model: app.db.models.Team },
                 { model: app.db.models.ProjectType },
                 { model: app.db.models.ProjectStack },
-                { model: app.db.models.ProjectTemplate }
+                { model: app.db.models.ProjectTemplate },
+                { model: app.db.models.ProjectSettings }
             ]
         })
-
         if (app.license.active() && app.billing) {
             await app.billing.initialiseProjectBillingState(team, project)
         }
@@ -786,6 +798,14 @@ module.exports = async function (app) {
             settings.env = Object.assign({}, settings.settings.env, settings.env)
             delete settings.settings.env
         }
+
+        if (app.config.features.enabled('ha')) {
+            const ha = await request.project.getHASettings()
+            if (ha && ha.replicas > 1) {
+                settings.ha = ha
+            }
+        }
+
         reply.send(settings)
     })
 
@@ -915,4 +935,10 @@ module.exports = async function (app) {
     function generateCredentialSecret () {
         return crypto.randomBytes(32).toString('hex')
     }
+
+    // app.get('/:projectId/ha', {
+    //     preHandler: app.needsPermission('project:read')
+    // }, async (request, reply) => {
+    //     reply.send(await request.project.getHASettings())
+    // })
 }

frontend/src/api/instances.js

@@ -136,6 +136,15 @@ const rollbackInstance = async (instanceId, snapshotId) => {
     }
     return client.post(`/api/v1/projects/${instanceId}/actions/rollback`, data).then(res => res.data)
 }
+
+const enableHAMode = async (instanceId) => {
+    const haConfig = { replicas: 2 }
+    return client.put(`/api/v1/projects/${instanceId}/ha`, haConfig)
+}
+const disableHAMode = async (instanceId) => {
+    return client.delete(`/api/v1/projects/${instanceId}/ha`)
+}
+
 export default {
     create,
     getInstance,
@@ -151,5 +160,7 @@ export default {
     getInstanceDevices,
     getInstanceDeviceSettings,
     updateInstanceDeviceSettings,
-    rollbackInstance
+    rollbackInstance,
+    enableHAMode,
+    disableHAMode
 }

frontend/src/components/StatusBadge.vue

@@ -34,7 +34,7 @@ import {
 } from '@heroicons/vue/outline'
 
 export default {
-    name: 'InstanceStatusBadge',
+    name: 'StatusBadge',
     components: {
         CloudDownloadIcon,
         CloudUploadIcon,

frontend/src/pages/application/Overview.vue

@@ -141,8 +141,8 @@ export default {
             return this.instances.map((instance) => {
                 instance.running = instance.meta?.state === 'running'
                 instance.notSuspended = instance.meta?.state !== 'suspended'
-
-                instance.disabled = !instance.running || this.isVisitingAdmin
+                instance.isHA = instance.ha?.replicas !== undefined
+                instance.disabled = !instance.running || this.isVisitingAdmin || instance.isHA
 
                 return instance
             })

frontend/src/pages/application/createInstance.vue

@@ -134,6 +134,10 @@ export default {
                     options: { ...copyParts }
                 }
             }
+            if (this.features.ha && createPayload.isHA) {
+                createPayload.ha = { replicas: 2 }
+            }
+            delete createPayload.isHA
 
             return instanceApi.create(createPayload)
         }