Implement API for Device Groups

docs/user/concepts.md

@@ -157,3 +157,12 @@ the remote device.
 To further simplify device registration, Provisioning Tokens can be created to allow 
 devices to automatically connect to a team without having to manually register them first.
 The token can also be configured to assign a device directly to a Node-RED instance within the team.
+
+### Device Groups
+
+**Introduced in FlowFuse 1.15**
+
+Device groups allow you to organise your Application devices into logical groups. 
+These groups can be the target of [DevOps Pipelines](#devops-pipeline) greatly simplifying
+the deployments to one or hundreds of devices.
+

forge/auditLog/application.js

@@ -33,6 +33,20 @@ module.exports = {
                         await log('application.device.snapshot.device-target-set', actionedBy, application?.id, generateBody({ error, device, snapshot }))
                     }
                 }
+            },
+            deviceGroup: {
+                async created (actionedBy, error, application, deviceGroup) {
+                    await log('application.deviceGroup.created', actionedBy, application?.id, generateBody({ error, application, deviceGroup }))
+                },
+                async updated (actionedBy, error, application, deviceGroup, updates) {
+                    await log('application.deviceGroup.updated', actionedBy, application?.id, generateBody({ error, application, deviceGroup, updates }))
+                },
+                async deleted (actionedBy, error, application, deviceGroup) {
+                    await log('application.deviceGroup.deleted', actionedBy, application?.id, generateBody({ error, application, deviceGroup }))
+                },
+                async membersChanged (actionedBy, error, application, deviceGroup, updates, info) {
+                    await log('application.deviceGroup.members.changed', actionedBy, application?.id, generateBody({ error, application, deviceGroup, updates, info }))
+                }
             }
         }
 

forge/auditLog/formatters.js

@@ -7,10 +7,10 @@ const isObject = (obj) => {
 /**
  * Generate a standard format body for the audit log display and database.
  * Any items null or missing must not generate a property in the body
- * @param {{ error?, team?, project?, sourceProject?, targetProject?, device?, sourceDevice?, targetDevice?, user?, stack?, billingSession?, subscription?, license?, updates?, snapshot?, role?, projectType?, info? } == {}} objects objects to include in body
- * @returns {{ error?, team?, project?, sourceProject?, targetProject?, device?, user?, stack?, billingSession?, subscription?, license?, updates?, snapshot?, role?, projectType? info? }
+ * @param {{ error?, team?, project?, sourceProject?, targetProject?, device?, sourceDevice?, targetDevice?, user?, stack?, billingSession?, subscription?, license?, updates?, snapshot?, pipeline?, pipelineStage?, pipelineStageTarget?, role?, projectType?, info?, deviceGroup?, interval?, threshold? } == {}} objects objects to include in body
+ * @returns {{ error?, team?, project?, sourceProject?, targetProject?, device?, user?, stack?, billingSession?, subscription?, license?, updates?, snapshot?, pipeline?, pipelineStage?, pipelineStageTarget?, role?, projectType? info?, deviceGroup?, interval?, threshold? }}
  */
-const generateBody = ({ error, team, application, project, sourceProject, targetProject, device, sourceDevice, targetDevice, user, stack, billingSession, subscription, license, updates, snapshot, pipeline, pipelineStage, role, projectType, info, interval, threshold } = {}) => {
+const generateBody = ({ error, team, application, project, sourceProject, targetProject, device, sourceDevice, targetDevice, user, stack, billingSession, subscription, license, updates, snapshot, pipeline, pipelineStage, pipelineStageTarget, role, projectType, info, deviceGroup, interval, threshold } = {}) => {
     const body = {}
 
     if (isObject(error) || typeof error === 'string') {
@@ -69,6 +69,9 @@ const generateBody = ({ error, team, application, project, sourceProject, target
     if (isObject(pipelineStage)) {
         body.pipelineStage = pipelineStageObject(pipelineStage)
     }
+    if (isObject(pipelineStageTarget)) {
+        body.pipelineStageTarget = pipelineStageObject(pipelineStageTarget)
+    }
     if (isObject(role) || typeof role === 'number') {
         body.role = roleObject(role)
     }
@@ -80,6 +83,9 @@ const generateBody = ({ error, team, application, project, sourceProject, target
     } else if (isStringWithLength(info)) {
         body.info = { info }
     }
+    if (isObject(deviceGroup)) {
+        body.deviceGroup = deviceGroupObject(deviceGroup)
+    }
 
     if (interval) {
         body.interval = interval
@@ -146,12 +152,14 @@ const formatLogEntry = (auditLogDbRow) => {
                 snapshot: body?.snapshot,
                 updates: body?.updates,
                 device: body?.device,
+                deviceGroup: body?.deviceGroup,
                 sourceDevice: body?.sourceDevice,
                 targetDevice: body?.targetDevice,
                 projectType: body?.projectType,
                 info: body?.info,
                 pipeline: body?.pipeline,
                 pipelineStage: body?.pipelineStage,
+                pipelineStageTarget: body?.pipelineStageTarget,
                 interval: body?.interval,
                 threshold: body?.threshold
             })
@@ -306,6 +314,13 @@ const projectTypeObject = (projectType) => {
         name: projectType?.name || null
     }
 }
+const deviceGroupObject = (deviceGroup) => {
+    return {
+        id: deviceGroup?.id || null,
+        hashid: deviceGroup?.hashid || null,
+        name: deviceGroup?.name || null
+    }
+}
 /**
  * Generates the `trigger` part of the audit log report
  * @param {object|number|'system'} actionedBy A user object or a user id. NOTE: 0 or 'system' can be used to indicate "system" triggered the event
@@ -573,6 +588,7 @@ module.exports = {
     teamObject,
     projectObject,
     deviceObject,
+    deviceGroupObject,
     userObject,
     stackObject,
     billingSessionObject,

forge/auditLog/team.js

@@ -149,6 +149,9 @@ module.exports = {
                 },
                 async stageAdded (actionedBy, error, team, application, pipeline, pipelineStage) {
                     await log('application.pipeline.stage-added', actionedBy, team?.id, generateBody({ error, team, application, pipeline, pipelineStage }))
+                },
+                async stageDeployed (actionedBy, error, team, application, pipeline, sourcePipelineStage, targetPipelineStage) {
+                    await log('application.pipeline.stage-deployed', actionedBy, team?.id, generateBody({ error, team, application, pipeline, pipelineStage: sourcePipelineStage, pipelineStageTarget: targetPipelineStage }))
                 }
             }
         }

forge/db/migrations/20231220-01-add-device-devicegroup.js

@@ -0,0 +1,62 @@
+/* eslint-disable no-unused-vars */
+/**
+ * Add DeviceGroups table that has an FK association with Applications and Devices
+ */
+
+const { Sequelize, QueryInterface } = require('sequelize')
+
+module.exports = {
+    /**
+     * upgrade database
+     * @param {QueryInterface} context QueryInterface
+     */
+    up: async (context) => {
+        await context.createTable('DeviceGroups', {
+            id: {
+                allowNull: false,
+                autoIncrement: true,
+                primaryKey: true,
+                type: Sequelize.INTEGER
+            },
+            name: {
+                type: Sequelize.STRING(255),
+                allowNull: false
+            },
+            description: {
+                type: Sequelize.TEXT
+            },
+            createdAt: {
+                allowNull: false,
+                type: Sequelize.DATE
+            },
+            updatedAt: {
+                allowNull: false,
+                type: Sequelize.DATE
+            },
+            ApplicationId: {
+                type: Sequelize.INTEGER,
+                references: {
+                    model: 'Applications',
+                    key: 'id'
+                },
+                onDelete: 'CASCADE',
+                onUpdate: 'CASCADE'
+            }
+        })
+
+        // add a column to Devices table that references DeviceGroups
+        await context.addColumn('Devices', 'DeviceGroupId', {
+            type: Sequelize.INTEGER,
+            references: {
+                model: 'DeviceGroups',
+                key: 'id'
+            },
+            onUpdate: 'CASCADE',
+            onDelete: 'SET NULL'
+        })
+    },
+
+    down: async (queryInterface, Sequelize) => {
+
+    }
+}

forge/db/models/Application.js

@@ -26,6 +26,7 @@ module.exports = {
         this.hasMany(M.Project)
         this.hasMany(M.Project, { as: 'Instances' })
         this.belongsTo(M.Team, { foreignKey: { allowNull: false } })
+        this.hasMany(M.DeviceGroup, { onDelete: 'CASCADE' })
     },
     finders: function (M) {
         return {

forge/db/models/Device.js

@@ -28,7 +28,7 @@ module.exports = {
         ownerType: {
             type: DataTypes.VIRTUAL(DataTypes.ENUM('instance', 'application', null)),
             get () {
-                return this.Project?.id ? 'instance' : (this.Application?.hashid ? 'application' : null)
+                return this.ProjectId ? 'instance' : (this.ApplicationId ? 'application' : null)
             }
         },
         isApplicationOwned: {
@@ -53,6 +53,7 @@ module.exports = {
         this.belongsTo(M.ProjectSnapshot, { as: 'activeSnapshot' })
         this.hasMany(M.DeviceSettings)
         this.hasMany(M.ProjectSnapshot) // associate device at application level with snapshots
+        this.belongsTo(M.DeviceGroup, { foreignKey: { allowNull: true } }) // SEE: forge/db/models/DeviceGroup.js for the other side of this relationship
     },
     hooks: function (M, app) {
         return {
@@ -208,7 +209,7 @@ module.exports = {
                         ]
                     })
                 },
-                getAll: async (pagination = {}, where = {}, { includeInstanceApplication = false } = {}) => {
+                getAll: async (pagination = {}, where = {}, { includeInstanceApplication = false, includeDeviceGroup = false } = {}) => {
                     // Pagination
                     const limit = Math.min(parseInt(pagination.limit) || 100, 100)
                     if (pagination.cursor) {
@@ -307,6 +308,12 @@ module.exports = {
                         }
                     ]
 
+                    if (includeDeviceGroup) {
+                        includes.push({
+                            model: M.DeviceGroup,
+                            attributes: ['hashid', 'id', 'name', 'description', 'ApplicationId']
+                        })
+                    }
                     const statusOnlyIncludes = projectInclude.include?.where ? [projectInclude] : []
 
                     const [rows, count] = await Promise.all([

forge/db/models/DeviceGroup.js

@@ -0,0 +1,117 @@
+/**
+ * A DeviceGroup.
+ * A logical grouping of devices for the primary intent of group deployments in the pipeline stages.
+ * @namespace forge.db.models.DeviceGroup
+ */
+
+const { DataTypes, literal } = require('sequelize')
+
+const { buildPaginationSearchClause } = require('../utils')
+const nameValidator = { msg: 'Device Group name cannot be empty' }
+
+module.exports = {
+    name: 'DeviceGroup',
+    schema: {
+        name: {
+            type: DataTypes.STRING,
+            allowNull: false,
+            validate: {
+                notEmpty: nameValidator,
+                notNull: nameValidator
+            }
+        },
+        description: { type: DataTypes.TEXT }
+    },
+    associations: function (M) {
+        this.belongsTo(M.Application, { onDelete: 'CASCADE' })
+        this.hasMany(M.Device)
+    },
+    finders: function (M) {
+        const self = this
+        const deviceCountLiteral = literal(`(
+            SELECT COUNT(*)
+            FROM "Devices" AS "device"
+            WHERE
+            "device"."DeviceGroupId" = "DeviceGroup"."id"
+            AND
+            "device"."ApplicationId" = "DeviceGroup"."ApplicationId"    
+        )`)
+        return {
+            static: {
+                byId: async function (id) {
+                    if (typeof id === 'string') {
+                        id = M.DeviceGroup.decodeHashid(id)
+                    }
+                    // find one, include application, devices and device count
+                    return self.findOne({
+                        where: { id },
+                        include: [
+                            { model: M.Application, attributes: ['hashid', 'id', 'name', 'TeamId'] },
+                            {
+                                model: M.Device,
+                                attributes: ['hashid', 'id', 'name', 'type', 'TeamId', 'ApplicationId', 'ProjectId', 'ownerType'],
+                                where: {
+                                    ApplicationId: literal('"Devices"."ApplicationId" = "Application"."id"')
+                                },
+                                required: false
+                            }
+                        ],
+                        attributes: {
+                            include: [
+                                [
+                                    deviceCountLiteral,
+                                    'deviceCount'
+                                ]
+                            ]
+                        }
+                    })
+                },
+                getAll: async (pagination = {}, where = {}) => {
+                    const limit = parseInt(pagination.limit) || 1000
+                    if (pagination.cursor) {
+                        pagination.cursor = M.DeviceGroup.decodeHashid(pagination.cursor)
+                    }
+                    if (where.ApplicationId && typeof where.ApplicationId === 'string') {
+                        where.ApplicationId = M.Application.decodeHashid(where.ApplicationId)
+                    }
+                    const [rows, count] = await Promise.all([
+                        this.findAll({
+                            where: buildPaginationSearchClause(pagination, where, ['DeviceGroup.name', 'DeviceGroup.description']),
+                            attributes: {
+                                include: [
+                                    [
+                                        deviceCountLiteral,
+                                        'deviceCount'
+                                    ]
+                                ]
+                            },
+                            order: [['id', 'ASC']],
+                            limit
+                        }),
+                        this.count({ where })
+                    ])
+                    return {
+                        meta: {
+                            next_cursor: rows.length === limit ? rows[rows.length - 1].hashid : undefined
+                        },
+                        count,
+                        groups: rows
+                    }
+                }
+            },
+            instance: {
+                deviceCount: async function () {
+                    return await M.Device.count({ where: { DeviceGroupId: this.id, ApplicationId: this.ApplicationId } })
+                },
+                getDevices: async function () {
+                    return await M.Device.findAll({
+                        where: {
+                            DeviceGroupId: this.id,
+                            ApplicationId: this.ApplicationId
+                        }
+                    })
+                }
+            }
+        }
+    }
+}

forge/db/models/index.js

@@ -69,6 +69,7 @@ const modelTypes = [
     'AccessToken',
     'AuthClient',
     'Device',
+    'DeviceGroup',
     'DeviceSettings',
     'StorageFlow',
     'StorageCredentials',

forge/db/views/Device.js

@@ -27,7 +27,11 @@ module.exports = function (app) {
             team: { $ref: 'TeamSummary' },
             instance: { $ref: 'InstanceSummary' },
             application: { $ref: 'ApplicationSummary' },
-            editor: { type: 'object', additionalProperties: true }
+            editor: { type: 'object', additionalProperties: true },
+            deviceGroup: {
+                nullable: true,
+                allOf: [{ $ref: 'DeviceGroupSummary' }]
+            }
         }
     })
 
@@ -36,7 +40,7 @@ module.exports = function (app) {
             return null
         }
 
-        const result = device.toJSON()
+        const result = device.toJSON ? device.toJSON() : device
 
         if (statusOnly) {
             return {
@@ -64,7 +68,8 @@ module.exports = function (app) {
             agentVersion: result.agentVersion,
             mode: result.mode || 'autonomous',
             ownerType: result.ownerType,
-            isDeploying: app.db.controllers.Device.isDeploying(device)
+            isDeploying: app.db.controllers.Device.isDeploying(device),
+            deviceGroup: device.DeviceGroup && app.db.views.DeviceGroup.deviceGroupSummary(device.DeviceGroup)
         }
         if (device.Team) {
             filtered.team = app.db.views.Team.teamSummary(device.Team)
@@ -104,7 +109,7 @@ module.exports = function (app) {
     })
     function deviceSummary (device, { includeSnapshotIds = false } = {}) {
         if (device) {
-            const result = device.toJSON()
+            const result = device.toJSON ? device.toJSON() : device
             const filtered = {
                 id: result.hashid,
                 ownerType: result.ownerType,