diff --git a/forge/db/models/Application.js b/forge/db/models/Application.js
index 6f1a430020..7755046962 100644
--- a/forge/db/models/Application.js
+++ b/forge/db/models/Application.js
@@ -2,9 +2,9 @@
* An application definition
* @namespace forge.db.models.Application
*/
-const { DataTypes } = require('sequelize')
+const { DataTypes, Op } = require('sequelize')
-const { KEY_SETTINGS } = require('./ProjectSettings')
+const { KEY_SETTINGS, KEY_HA } = require('./ProjectSettings')
module.exports = {
name: 'Application',
@@ -60,7 +60,12 @@ module.exports = {
attributes: ['hashid', 'id', 'name', 'links', 'settings', 'policy']
}, {
model: M.ProjectSettings,
- where: { key: KEY_SETTINGS },
+ where: {
+ [Op.or]: [
+ { key: KEY_SETTINGS },
+ { key: KEY_HA }
+ ]
+ },
required: false
}
]
diff --git a/forge/db/models/Project.js b/forge/db/models/Project.js
index 2c467598db..17fe2fe756 100644
--- a/forge/db/models/Project.js
+++ b/forge/db/models/Project.js
@@ -14,7 +14,7 @@
const { DataTypes, Op } = require('sequelize')
const Controllers = require('../controllers')
-const { KEY_HOSTNAME, KEY_SETTINGS } = require('./ProjectSettings')
+const { KEY_HOSTNAME, KEY_SETTINGS, KEY_HA } = require('./ProjectSettings')
/** @type {FFModel} */
module.exports = {
@@ -207,7 +207,7 @@ module.exports = {
if (key === 'settings' && value && Array.isArray(value.env)) {
value.env = Controllers.Project.removePlatformSpecificEnvVars(value.env) // remove platform specific values
}
- return await M.ProjectSettings.upsert({ ProjectId: this.id, key, value }, options)
+ return M.ProjectSettings.upsert({ ProjectId: this.id, key, value }, options)
},
async getSetting (key) {
const result = await M.ProjectSettings.findOne({ where: { ProjectId: this.id, key } })
@@ -220,7 +220,9 @@ module.exports = {
}
return undefined
},
-
+ async removeSetting (key, options) {
+ return M.ProjectSettings.destroy({ where: { ProjectId: this.id, key } }, options)
+ },
async getCredentialSecret () {
// If this project was created at 0.6+ but then started with a <0.6 launcher
// (for example, in k8s with an old stack) then the project will have both
@@ -335,7 +337,8 @@ module.exports = {
where: {
[Op.or]: [
{ key: KEY_SETTINGS },
- { key: KEY_HOSTNAME }
+ { key: KEY_HOSTNAME },
+ { key: KEY_HA }
]
},
required: false
@@ -371,7 +374,12 @@ module.exports = {
include.push({
model: M.ProjectSettings,
- where: { key: KEY_SETTINGS },
+ where: {
+ [Op.or]: [
+ { key: KEY_SETTINGS },
+ { key: KEY_HA }
+ ]
+ },
required: false
})
}
diff --git a/forge/db/models/ProjectSettings.js b/forge/db/models/ProjectSettings.js
index 01308a881a..07d2d5eb0a 100644
--- a/forge/db/models/ProjectSettings.js
+++ b/forge/db/models/ProjectSettings.js
@@ -12,6 +12,7 @@ const SettingTypes = {
const KEY_SETTINGS = 'settings'
const KEY_HOSTNAME = 'hostname'
const KEY_BILLING_STATE = 'billingState'
+const KEY_HA = 'ha'
const BILLING_STATES = {
UNKNOWN: undefined,
@@ -25,6 +26,7 @@ module.exports = {
KEY_SETTINGS,
KEY_HOSTNAME,
KEY_BILLING_STATE,
+ KEY_HA,
name: 'ProjectSettings',
schema: {
ProjectId: { type: DataTypes.UUID, unique: 'pk_settings' },
diff --git a/forge/db/views/Project.js b/forge/db/views/Project.js
index 6224790468..35d03b6249 100644
--- a/forge/db/views/Project.js
+++ b/forge/db/views/Project.js
@@ -1,4 +1,4 @@
-const { KEY_HOSTNAME, KEY_SETTINGS } = require('../models/ProjectSettings')
+const { KEY_HOSTNAME, KEY_SETTINGS, KEY_HA } = require('../models/ProjectSettings')
module.exports = {
project: async function (app, project, { includeSettings = true } = {}) {
@@ -35,6 +35,10 @@ module.exports = {
const settingsHostnameRow = proj.ProjectSettings?.find((projectSettingsRow) => projectSettingsRow.key === KEY_HOSTNAME)
result.hostname = settingsHostnameRow?.value || ''
}
+ if (app.config.features.enabled('ha')) {
+ const settingsHARow = proj.ProjectSettings?.find(row => row.key === KEY_HA)
+ result.ha = settingsHARow?.value || { disabled: true }
+ }
if (proj.Application) {
result.application = app.db.views.Application.applicationSummary(proj.Application)
diff --git a/forge/ee/lib/ha/index.js b/forge/ee/lib/ha/index.js
new file mode 100644
index 0000000000..fba33b75ac
--- /dev/null
+++ b/forge/ee/lib/ha/index.js
@@ -0,0 +1,41 @@
+const { KEY_HA } = require('../../../db/models/ProjectSettings')
+
+module.exports.init = function (app) {
+ if (app.config.driver.type === 'k8s' || app.config.driver.type === 'stub') {
+ // Register ha flag as a private flag - no requirement to expose it in public settings
+ app.config.features.register('ha', true)
+
+ /**
+ * Check if HA is allowed for this given team/projectType/haConfig combination
+ * @param {*} team
+ * @param {*} projectType
+ * @param {*} haConfig
+ * @returns true/false
+ */
+ async function isHAAllowed (team, projectType, haConfig) {
+ // For initial beta release, we will support 1-2 replicas.
+ // 1 replica is equivalent to no HA
+ // In the future this will need to take into account the team type
+ return (haConfig.replicas > 0 && haConfig.replicas < 3)
+ }
+
+ // Add ha functions to the Project model
+ app.db.models.Project.prototype.getHASettings = async function () {
+ return this.getSetting(KEY_HA)
+ }
+ app.db.models.Project.prototype.updateHASettings = async function (haConfig) {
+ if (!haConfig) {
+ return this.removeSetting(KEY_HA)
+ }
+ if (haConfig?.replicas > 0 && haConfig?.replicas < 3) {
+ return this.updateSetting(KEY_HA, {
+ replicas: haConfig.replicas
+ })
+ }
+ }
+
+ app.decorate('ha', {
+ isHAAllowed
+ })
+ }
+}
diff --git a/forge/ee/lib/index.js b/forge/ee/lib/index.js
index 5f51d59e8a..fab73311db 100644
--- a/forge/ee/lib/index.js
+++ b/forge/ee/lib/index.js
@@ -6,6 +6,7 @@ module.exports = fp(async function (app, opts, done) {
}
require('./projectComms').init(app)
require('./deviceEditor').init(app)
+ require('./ha').init(app)
app.decorate('sso', await require('./sso').init(app))
diff --git a/forge/ee/routes/ha/index.js b/forge/ee/routes/ha/index.js
new file mode 100644
index 0000000000..78b562e511
--- /dev/null
+++ b/forge/ee/routes/ha/index.js
@@ -0,0 +1,107 @@
+module.exports = async function (app) {
+ app.addHook('preHandler', app.verifySession)
+ app.addHook('preHandler', async (request, reply) => {
+ if (!app.config.features.enabled('ha')) {
+ reply.code(404).send({ code: 'not_found', error: 'Not Found' })
+ }
+ })
+ app.addHook('preHandler', async (request, reply) => {
+ if (request.params.projectId !== undefined) {
+ if (request.params.projectId) {
+ try {
+ request.project = await app.db.models.Project.byId(request.params.projectId)
+ if (!request.project) {
+ reply.code(404).send({ code: 'not_found', error: 'Not Found' })
+ return
+ }
+ if (request.session.User) {
+ request.teamMembership = await request.session.User.getTeamMembership(request.project.Team.id)
+ if (!request.teamMembership && !request.session.User.admin) {
+ reply.code(404).send({ code: 'not_found', error: 'Not Found' })
+ return
+ }
+ } else if (request.session.ownerId !== request.params.projectId) {
+ // AccesToken being used - but not owned by this project
+ reply.code(404).send({ code: 'not_found', error: 'Not Found' })
+ return
+ }
+ } catch (err) {
+ reply.code(404).send({ code: 'not_found', error: 'Not Found' })
+ }
+ } else {
+ reply.code(404).send({ code: 'not_found', error: 'Not Found' })
+ }
+ }
+ })
+
+ app.get('/', {
+ preHandler: app.needsPermission('project:read')
+ }, async (request, reply) => {
+ reply.send(await request.project.getHASettings() || {})
+ })
+
+ app.put('/', {
+ preHandler: app.needsPermission('project:edit')
+ }, async (request, reply) => {
+ // For 1.8, only support setting replica count to 2
+ if (request.body.replicas !== 2) {
+ reply.code(409).send({ code: 'invalid_ha_configuration', error: 'Invalid HA configuration -only 2 replicas are allowed' })
+ return
+ }
+ const existingHA = await request.project.getHASettings() || {}
+ if (existingHA.replicas !== request.body.replicas) {
+ // This is a change in replica count.
+ await request.project.updateHASettings({ replicas: request.body.replicas })
+ await applyUpdatedInstanceSettings(reply, request.project, request.session.User)
+ } else {
+ reply.send(await request.project.getHASettings() || {})
+ }
+ })
+
+ app.delete('/', {
+ preHandler: app.needsPermission('project:edit')
+ }, async (request, reply) => {
+ const existingHA = await request.project.getHASettings() || {}
+ if (existingHA.replicas) {
+ // This instance already has ha configured - clear the setting
+ // and apply
+ await request.project.updateHASettings(undefined)
+ await applyUpdatedInstanceSettings(reply, request.project, request.session.User)
+ } else {
+ reply.send({})
+ }
+ })
+
+ async function applyUpdatedInstanceSettings (reply, project, user) {
+ if (project.state !== 'suspended') {
+ // This code is copy/paste with slight changes from projects.js
+ // We also have projectActions.js that does suspend logic.
+ // TODO: refactor into a Model function to suspend a project
+ app.db.controllers.Project.setInflightState(project, 'starting') // TODO: better inflight state needed
+ reply.send(await project.getHASettings() || {})
+
+ const targetState = project.state
+ app.log.info(`Stopping project ${project.id}`)
+ await app.containers.stop(project, {
+ skipBilling: true
+ })
+ await app.auditLog.Project.project.suspended(user, null, project)
+ app.log.info(`Restarting project ${project.id}`)
+ project.state = targetState
+ await project.save()
+ await project.reload()
+ const startResult = await app.containers.start(project)
+ startResult.started.then(async () => {
+ await app.auditLog.Project.project.started(user, null, project)
+ app.db.controllers.Project.clearInflightState(project)
+ return true
+ }).catch(_ => {
+ app.db.controllers.Project.clearInflightState(project)
+ })
+ } else {
+ // A suspended project doesn't need to do anything more
+ // The settings will get applied when it is next resumed
+ reply.send(await project.getHASettings() || {})
+ }
+ }
+}
diff --git a/forge/ee/routes/index.js b/forge/ee/routes/index.js
index 985f22289a..06109e679e 100644
--- a/forge/ee/routes/index.js
+++ b/forge/ee/routes/index.js
@@ -13,6 +13,8 @@ module.exports = async function (app) {
await app.register(require('./pipeline'), { prefix: '/api/v1', logLevel: app.config.logging.http })
await app.register(require('./deviceEditor'), { prefix: '/api/v1/devices/:deviceId/editor', logLevel: app.config.logging.http })
+ await app.register(require('./ha'), { prefix: '/api/v1/projects/:projectId/ha', logLevel: app.config.logging.http })
+
// Important: keep SSO last to avoid its error handling polluting other routes.
await app.register(require('./sso'), { logLevel: app.config.logging.http })
}
diff --git a/forge/routes/api/project.js b/forge/routes/api/project.js
index 40be75dbb4..fb176f5073 100644
--- a/forge/routes/api/project.js
+++ b/forge/routes/api/project.js
@@ -193,6 +193,13 @@ module.exports = async function (app) {
return
}
+ if (app.license.active() && app.ha) {
+ if (request.body.ha && !await app.ha.isHAAllowed(team, projectType, request.body.ha)) {
+ reply.code(400).send({ code: 'invalid_ha', error: 'Invalid HA configuration' })
+ return
+ }
+ }
+
let project
try {
project = await app.db.models.Project.create({
@@ -212,15 +219,20 @@ module.exports = async function (app) {
await project.setProjectStack(stack)
await project.setProjectTemplate(template)
await project.setProjectType(projectType)
+
+ if (app.license.active() && app.ha && request.body.ha) {
+ await project.updateHASettings(request.body.ha)
+ }
+
await project.reload({
include: [
{ model: app.db.models.Team },
{ model: app.db.models.ProjectType },
{ model: app.db.models.ProjectStack },
- { model: app.db.models.ProjectTemplate }
+ { model: app.db.models.ProjectTemplate },
+ { model: app.db.models.ProjectSettings }
]
})
-
if (app.license.active() && app.billing) {
await app.billing.initialiseProjectBillingState(team, project)
}
@@ -786,6 +798,14 @@ module.exports = async function (app) {
settings.env = Object.assign({}, settings.settings.env, settings.env)
delete settings.settings.env
}
+
+ if (app.config.features.enabled('ha')) {
+ const ha = await request.project.getHASettings()
+ if (ha && ha.replicas > 1) {
+ settings.ha = ha
+ }
+ }
+
reply.send(settings)
})
@@ -915,4 +935,10 @@ module.exports = async function (app) {
function generateCredentialSecret () {
return crypto.randomBytes(32).toString('hex')
}
+
+ // app.get('/:projectId/ha', {
+ // preHandler: app.needsPermission('project:read')
+ // }, async (request, reply) => {
+ // reply.send(await request.project.getHASettings())
+ // })
}
diff --git a/frontend/src/api/instances.js b/frontend/src/api/instances.js
index 0c4b160fca..b9c4542438 100644
--- a/frontend/src/api/instances.js
+++ b/frontend/src/api/instances.js
@@ -136,6 +136,15 @@ const rollbackInstance = async (instanceId, snapshotId) => {
}
return client.post(`/api/v1/projects/${instanceId}/actions/rollback`, data).then(res => res.data)
}
+
+const enableHAMode = async (instanceId) => {
+ const haConfig = { replicas: 2 }
+ return client.put(`/api/v1/projects/${instanceId}/ha`, haConfig)
+}
+const disableHAMode = async (instanceId) => {
+ return client.delete(`/api/v1/projects/${instanceId}/ha`)
+}
+
export default {
create,
getInstance,
@@ -151,5 +160,7 @@ export default {
getInstanceDevices,
getInstanceDeviceSettings,
updateInstanceDeviceSettings,
- rollbackInstance
+ rollbackInstance,
+ enableHAMode,
+ disableHAMode
}
diff --git a/frontend/src/components/StatusBadge.vue b/frontend/src/components/StatusBadge.vue
index 03b7f89526..e8de886335 100644
--- a/frontend/src/components/StatusBadge.vue
+++ b/frontend/src/components/StatusBadge.vue
@@ -34,7 +34,7 @@ import {
} from '@heroicons/vue/outline'
export default {
- name: 'InstanceStatusBadge',
+ name: 'StatusBadge',
components: {
CloudDownloadIcon,
CloudUploadIcon,
diff --git a/frontend/src/pages/application/Overview.vue b/frontend/src/pages/application/Overview.vue
index 54ccfb8914..f0484122ca 100644
--- a/frontend/src/pages/application/Overview.vue
+++ b/frontend/src/pages/application/Overview.vue
@@ -141,8 +141,8 @@ export default {
return this.instances.map((instance) => {
instance.running = instance.meta?.state === 'running'
instance.notSuspended = instance.meta?.state !== 'suspended'
-
- instance.disabled = !instance.running || this.isVisitingAdmin
+ instance.isHA = instance.ha?.replicas !== undefined
+ instance.disabled = !instance.running || this.isVisitingAdmin || instance.isHA
return instance
})
diff --git a/frontend/src/pages/application/createInstance.vue b/frontend/src/pages/application/createInstance.vue
index 92f5a0be2c..e44eda608b 100644
--- a/frontend/src/pages/application/createInstance.vue
+++ b/frontend/src/pages/application/createInstance.vue
@@ -134,6 +134,10 @@ export default {
options: { ...copyParts }
}
}
+ if (this.features.ha && createPayload.isHA) {
+ createPayload.ha = { replicas: 2 }
+ }
+ delete createPayload.isHA
return instanceApi.create(createPayload)
}
diff --git a/frontend/src/pages/instance/Overview.vue b/frontend/src/pages/instance/Overview.vue
index b32562e787..79dee145e2 100644
--- a/frontend/src/pages/instance/Overview.vue
+++ b/frontend/src/pages/instance/Overview.vue
@@ -17,7 +17,14 @@