Skip to content

Commit

Permalink
Merge pull request #3157 from FlowFuse/2997-device-groups-backend
Browse files Browse the repository at this point in the history
Implement API for Device Groups
  • Loading branch information
knolleary committed Dec 20, 2023
2 parents cf07c7a + 4796eca commit fb33603
Show file tree
Hide file tree
Showing 63 changed files with 4,358 additions and 232 deletions.
9 changes: 9 additions & 0 deletions docs/user/concepts.md
Original file line number Diff line number Diff line change
Expand Up @@ -157,3 +157,12 @@ the remote device.
To further simplify device registration, Provisioning Tokens can be created to allow
devices to automatically connect to a team without having to manually register them first.
The token can also be configured to assign a device directly to a Node-RED instance within the team.

### Device Groups

**Introduced in FlowFuse 1.15**

Device groups allow you to organise your Application devices into logical groups.
These groups can be the target of [DevOps Pipelines](#devops-pipeline) greatly simplifying
the deployments to one or hundreds of devices.

14 changes: 14 additions & 0 deletions forge/auditLog/application.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,20 @@ module.exports = {
await log('application.device.snapshot.device-target-set', actionedBy, application?.id, generateBody({ error, device, snapshot }))
}
}
},
deviceGroup: {
async created (actionedBy, error, application, deviceGroup) {
await log('application.deviceGroup.created', actionedBy, application?.id, generateBody({ error, application, deviceGroup }))
},
async updated (actionedBy, error, application, deviceGroup, updates) {
await log('application.deviceGroup.updated', actionedBy, application?.id, generateBody({ error, application, deviceGroup, updates }))
},
async deleted (actionedBy, error, application, deviceGroup) {
await log('application.deviceGroup.deleted', actionedBy, application?.id, generateBody({ error, application, deviceGroup }))
},
async membersChanged (actionedBy, error, application, deviceGroup, updates, info) {
await log('application.deviceGroup.members.changed', actionedBy, application?.id, generateBody({ error, application, deviceGroup, updates, info }))
}
}
}

Expand Down
22 changes: 19 additions & 3 deletions forge/auditLog/formatters.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ const isObject = (obj) => {
/**
* Generate a standard format body for the audit log display and database.
* Any items null or missing must not generate a property in the body
* @param {{ error?, team?, project?, sourceProject?, targetProject?, device?, sourceDevice?, targetDevice?, user?, stack?, billingSession?, subscription?, license?, updates?, snapshot?, role?, projectType?, info? } == {}} objects objects to include in body
* @returns {{ error?, team?, project?, sourceProject?, targetProject?, device?, user?, stack?, billingSession?, subscription?, license?, updates?, snapshot?, role?, projectType? info? }
* @param {{ error?, team?, project?, sourceProject?, targetProject?, device?, sourceDevice?, targetDevice?, user?, stack?, billingSession?, subscription?, license?, updates?, snapshot?, pipeline?, pipelineStage?, pipelineStageTarget?, role?, projectType?, info?, deviceGroup?, interval?, threshold? } == {}} objects objects to include in body
* @returns {{ error?, team?, project?, sourceProject?, targetProject?, device?, user?, stack?, billingSession?, subscription?, license?, updates?, snapshot?, pipeline?, pipelineStage?, pipelineStageTarget?, role?, projectType? info?, deviceGroup?, interval?, threshold? }}
*/
const generateBody = ({ error, team, application, project, sourceProject, targetProject, device, sourceDevice, targetDevice, user, stack, billingSession, subscription, license, updates, snapshot, pipeline, pipelineStage, role, projectType, info, interval, threshold } = {}) => {
const generateBody = ({ error, team, application, project, sourceProject, targetProject, device, sourceDevice, targetDevice, user, stack, billingSession, subscription, license, updates, snapshot, pipeline, pipelineStage, pipelineStageTarget, role, projectType, info, deviceGroup, interval, threshold } = {}) => {
const body = {}

if (isObject(error) || typeof error === 'string') {
Expand Down Expand Up @@ -69,6 +69,9 @@ const generateBody = ({ error, team, application, project, sourceProject, target
if (isObject(pipelineStage)) {
body.pipelineStage = pipelineStageObject(pipelineStage)
}
if (isObject(pipelineStageTarget)) {
body.pipelineStageTarget = pipelineStageObject(pipelineStageTarget)
}
if (isObject(role) || typeof role === 'number') {
body.role = roleObject(role)
}
Expand All @@ -80,6 +83,9 @@ const generateBody = ({ error, team, application, project, sourceProject, target
} else if (isStringWithLength(info)) {
body.info = { info }
}
if (isObject(deviceGroup)) {
body.deviceGroup = deviceGroupObject(deviceGroup)
}

if (interval) {
body.interval = interval
Expand Down Expand Up @@ -146,12 +152,14 @@ const formatLogEntry = (auditLogDbRow) => {
snapshot: body?.snapshot,
updates: body?.updates,
device: body?.device,
deviceGroup: body?.deviceGroup,
sourceDevice: body?.sourceDevice,
targetDevice: body?.targetDevice,
projectType: body?.projectType,
info: body?.info,
pipeline: body?.pipeline,
pipelineStage: body?.pipelineStage,
pipelineStageTarget: body?.pipelineStageTarget,
interval: body?.interval,
threshold: body?.threshold
})
Expand Down Expand Up @@ -306,6 +314,13 @@ const projectTypeObject = (projectType) => {
name: projectType?.name || null
}
}
const deviceGroupObject = (deviceGroup) => {
return {
id: deviceGroup?.id || null,
hashid: deviceGroup?.hashid || null,
name: deviceGroup?.name || null
}
}
/**
* Generates the `trigger` part of the audit log report
* @param {object|number|'system'} actionedBy A user object or a user id. NOTE: 0 or 'system' can be used to indicate "system" triggered the event
Expand Down Expand Up @@ -573,6 +588,7 @@ module.exports = {
teamObject,
projectObject,
deviceObject,
deviceGroupObject,
userObject,
stackObject,
billingSessionObject,
Expand Down
3 changes: 3 additions & 0 deletions forge/auditLog/team.js
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,9 @@ module.exports = {
},
async stageAdded (actionedBy, error, team, application, pipeline, pipelineStage) {
await log('application.pipeline.stage-added', actionedBy, team?.id, generateBody({ error, team, application, pipeline, pipelineStage }))
},
async stageDeployed (actionedBy, error, team, application, pipeline, sourcePipelineStage, targetPipelineStage) {
await log('application.pipeline.stage-deployed', actionedBy, team?.id, generateBody({ error, team, application, pipeline, pipelineStage: sourcePipelineStage, pipelineStageTarget: targetPipelineStage }))
}
}
}
Expand Down
62 changes: 62 additions & 0 deletions forge/db/migrations/20231220-01-add-device-devicegroup.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/* eslint-disable no-unused-vars */
/**
* Add DeviceGroups table that has an FK association with Applications and Devices
*/

const { Sequelize, QueryInterface } = require('sequelize')

module.exports = {
/**
* upgrade database
* @param {QueryInterface} context QueryInterface
*/
up: async (context) => {
await context.createTable('DeviceGroups', {
id: {
allowNull: false,
autoIncrement: true,
primaryKey: true,
type: Sequelize.INTEGER
},
name: {
type: Sequelize.STRING(255),
allowNull: false
},
description: {
type: Sequelize.TEXT
},
createdAt: {
allowNull: false,
type: Sequelize.DATE
},
updatedAt: {
allowNull: false,
type: Sequelize.DATE
},
ApplicationId: {
type: Sequelize.INTEGER,
references: {
model: 'Applications',
key: 'id'
},
onDelete: 'CASCADE',
onUpdate: 'CASCADE'
}
})

// add a column to Devices table that references DeviceGroups
await context.addColumn('Devices', 'DeviceGroupId', {
type: Sequelize.INTEGER,
references: {
model: 'DeviceGroups',
key: 'id'
},
onUpdate: 'CASCADE',
onDelete: 'SET NULL'
})
},

down: async (queryInterface, Sequelize) => {

}
}
1 change: 1 addition & 0 deletions forge/db/models/Application.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ module.exports = {
this.hasMany(M.Project)
this.hasMany(M.Project, { as: 'Instances' })
this.belongsTo(M.Team, { foreignKey: { allowNull: false } })
this.hasMany(M.DeviceGroup, { onDelete: 'CASCADE' })
},
finders: function (M) {
return {
Expand Down
11 changes: 9 additions & 2 deletions forge/db/models/Device.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ module.exports = {
ownerType: {
type: DataTypes.VIRTUAL(DataTypes.ENUM('instance', 'application', null)),
get () {
return this.Project?.id ? 'instance' : (this.Application?.hashid ? 'application' : null)
return this.ProjectId ? 'instance' : (this.ApplicationId ? 'application' : null)
}
},
isApplicationOwned: {
Expand All @@ -53,6 +53,7 @@ module.exports = {
this.belongsTo(M.ProjectSnapshot, { as: 'activeSnapshot' })
this.hasMany(M.DeviceSettings)
this.hasMany(M.ProjectSnapshot) // associate device at application level with snapshots
this.belongsTo(M.DeviceGroup, { foreignKey: { allowNull: true } }) // SEE: forge/db/models/DeviceGroup.js for the other side of this relationship
},
hooks: function (M, app) {
return {
Expand Down Expand Up @@ -208,7 +209,7 @@ module.exports = {
]
})
},
getAll: async (pagination = {}, where = {}, { includeInstanceApplication = false } = {}) => {
getAll: async (pagination = {}, where = {}, { includeInstanceApplication = false, includeDeviceGroup = false } = {}) => {
// Pagination
const limit = Math.min(parseInt(pagination.limit) || 100, 100)
if (pagination.cursor) {
Expand Down Expand Up @@ -307,6 +308,12 @@ module.exports = {
}
]

if (includeDeviceGroup) {
includes.push({
model: M.DeviceGroup,
attributes: ['hashid', 'id', 'name', 'description', 'ApplicationId']
})
}
const statusOnlyIncludes = projectInclude.include?.where ? [projectInclude] : []

const [rows, count] = await Promise.all([
Expand Down
117 changes: 117 additions & 0 deletions forge/db/models/DeviceGroup.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/**
* A DeviceGroup.
* A logical grouping of devices for the primary intent of group deployments in the pipeline stages.
* @namespace forge.db.models.DeviceGroup
*/

const { DataTypes, literal } = require('sequelize')

const { buildPaginationSearchClause } = require('../utils')
const nameValidator = { msg: 'Device Group name cannot be empty' }

module.exports = {
name: 'DeviceGroup',
schema: {
name: {
type: DataTypes.STRING,
allowNull: false,
validate: {
notEmpty: nameValidator,
notNull: nameValidator
}
},
description: { type: DataTypes.TEXT }
},
associations: function (M) {
this.belongsTo(M.Application, { onDelete: 'CASCADE' })
this.hasMany(M.Device)
},
finders: function (M) {
const self = this
const deviceCountLiteral = literal(`(
SELECT COUNT(*)
FROM "Devices" AS "device"
WHERE
"device"."DeviceGroupId" = "DeviceGroup"."id"
AND
"device"."ApplicationId" = "DeviceGroup"."ApplicationId"
)`)
return {
static: {
byId: async function (id) {
if (typeof id === 'string') {
id = M.DeviceGroup.decodeHashid(id)
}
// find one, include application, devices and device count
return self.findOne({
where: { id },
include: [
{ model: M.Application, attributes: ['hashid', 'id', 'name', 'TeamId'] },
{
model: M.Device,
attributes: ['hashid', 'id', 'name', 'type', 'TeamId', 'ApplicationId', 'ProjectId', 'ownerType'],
where: {
ApplicationId: literal('"Devices"."ApplicationId" = "Application"."id"')
},
required: false
}
],
attributes: {
include: [
[
deviceCountLiteral,
'deviceCount'
]
]
}
})
},
getAll: async (pagination = {}, where = {}) => {
const limit = parseInt(pagination.limit) || 1000
if (pagination.cursor) {
pagination.cursor = M.DeviceGroup.decodeHashid(pagination.cursor)
}
if (where.ApplicationId && typeof where.ApplicationId === 'string') {
where.ApplicationId = M.Application.decodeHashid(where.ApplicationId)
}
const [rows, count] = await Promise.all([
this.findAll({
where: buildPaginationSearchClause(pagination, where, ['DeviceGroup.name', 'DeviceGroup.description']),
attributes: {
include: [
[
deviceCountLiteral,
'deviceCount'
]
]
},
order: [['id', 'ASC']],
limit
}),
this.count({ where })
])
return {
meta: {
next_cursor: rows.length === limit ? rows[rows.length - 1].hashid : undefined
},
count,
groups: rows
}
}
},
instance: {
deviceCount: async function () {
return await M.Device.count({ where: { DeviceGroupId: this.id, ApplicationId: this.ApplicationId } })
},
getDevices: async function () {
return await M.Device.findAll({
where: {
DeviceGroupId: this.id,
ApplicationId: this.ApplicationId
}
})
}
}
}
}
}
1 change: 1 addition & 0 deletions forge/db/models/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ const modelTypes = [
'AccessToken',
'AuthClient',
'Device',
'DeviceGroup',
'DeviceSettings',
'StorageFlow',
'StorageCredentials',
Expand Down
13 changes: 9 additions & 4 deletions forge/db/views/Device.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,11 @@ module.exports = function (app) {
team: { $ref: 'TeamSummary' },
instance: { $ref: 'InstanceSummary' },
application: { $ref: 'ApplicationSummary' },
editor: { type: 'object', additionalProperties: true }
editor: { type: 'object', additionalProperties: true },
deviceGroup: {
nullable: true,
allOf: [{ $ref: 'DeviceGroupSummary' }]
}
}
})

Expand All @@ -36,7 +40,7 @@ module.exports = function (app) {
return null
}

const result = device.toJSON()
const result = device.toJSON ? device.toJSON() : device

if (statusOnly) {
return {
Expand Down Expand Up @@ -64,7 +68,8 @@ module.exports = function (app) {
agentVersion: result.agentVersion,
mode: result.mode || 'autonomous',
ownerType: result.ownerType,
isDeploying: app.db.controllers.Device.isDeploying(device)
isDeploying: app.db.controllers.Device.isDeploying(device),
deviceGroup: device.DeviceGroup && app.db.views.DeviceGroup.deviceGroupSummary(device.DeviceGroup)
}
if (device.Team) {
filtered.team = app.db.views.Team.teamSummary(device.Team)
Expand Down Expand Up @@ -104,7 +109,7 @@ module.exports = function (app) {
})
function deviceSummary (device, { includeSnapshotIds = false } = {}) {
if (device) {
const result = device.toJSON()
const result = device.toJSON ? device.toJSON() : device
const filtered = {
id: result.hashid,
ownerType: result.ownerType,
Expand Down
Loading

0 comments on commit fb33603

Please sign in to comment.