From e9fc7d424838cbd95b9c7a77c2c3f8848b8c71a1 Mon Sep 17 00:00:00 2001 From: cstns Date: Thu, 16 Apr 2026 16:50:57 +0300 Subject: [PATCH 01/54] - add init client expert chat api client - alter the way we deduce immersive entities in such a way that we have full contextual awareness throughout the app - rough mqtt implementation --- frontend/src/api/user.js | 6 +- frontend/src/mixins/Application.js | 11 ++ frontend/src/mixins/Instance.js | 11 +- frontend/src/pages/application/index.vue | 4 - frontend/src/pages/device/Editor/index.vue | 7 +- frontend/src/pages/device/index.vue | 5 + frontend/src/pages/instance/Editor/index.vue | 12 +- frontend/src/stores/context.js | 6 +- frontend/src/stores/product-assistant.js | 12 +- .../stores/product-expert-insights-agent.js | 5 +- .../stores/product-expert-support-agent.js | 5 +- frontend/src/stores/product-expert.js | 173 +++++++++++++++++- 12 files changed, 235 insertions(+), 22 deletions(-) diff --git a/frontend/src/api/user.js b/frontend/src/api/user.js index 21bf7c906a..3f1c21c37d 100644 --- a/frontend/src/api/user.js +++ b/frontend/src/api/user.js @@ -239,6 +239,9 @@ const disableMFA = async () => { const verifyMFA = async (token) => { return client.put('/api/v1/user/mfa/verify', { token }).then(res => res.data) } +const initiateExpertChat = async ({ sessionId }) => { + return client.post('/api/v1/user/expert-creds', { sessionId }).then(res => res.data) +} export default { registerUser, @@ -267,5 +270,6 @@ export default { updatePersonalAccessToken, enableMFA, verifyMFA, - disableMFA + disableMFA, + initiateExpertChat } diff --git a/frontend/src/mixins/Application.js b/frontend/src/mixins/Application.js index e744829d7e..7812450172 100644 --- a/frontend/src/mixins/Application.js +++ b/frontend/src/mixins/Application.js @@ -1,3 +1,5 @@ +import { mapActions } from 'pinia' + import ApplicationApi from '../api/application.js' import alerts from '../services/alerts.js' @@ -26,8 +28,14 @@ export default { } }, watch: { + application (application) { + if (application) { + this.setContextualApplication(application) + } + } }, methods: { + ...mapActions(useContextStore, { setContextualApplication: 'setApplication' }), async updateApplication () { const applicationId = this.$route.params.id @@ -96,5 +104,8 @@ export default { }, async created () { + }, + beforeUnmount () { + this.setContextualApplication(null) } } diff --git a/frontend/src/mixins/Instance.js b/frontend/src/mixins/Instance.js index 99e2db4bbf..7a2d00b903 100644 --- a/frontend/src/mixins/Instance.js +++ b/frontend/src/mixins/Instance.js @@ -1,4 +1,4 @@ -import { mapState } from 'pinia' +import { mapActions, mapState } from 'pinia' import InstanceApi from '../api/instances.js' import SnapshotApi from '../api/projectSnapshots.js' @@ -34,9 +34,13 @@ export default { } }, watch: { - instance: 'instanceChanged' + instance (instance) { + this.instanceChanged() + this.setContextualInstance(instance) + } }, methods: { + ...mapActions(useContextStore, { setContextualInstance: 'setInstance' }), showConfirmDeleteDialog () { this.$refs.confirmInstanceDeleteDialog.show(this.instance) }, @@ -106,5 +110,8 @@ export default { await this.loadInstance() } ) + }, + beforeUnmount () { + this.setContextualInstance(null) } } diff --git a/frontend/src/pages/application/index.vue b/frontend/src/pages/application/index.vue index 3b5b649626..9d63f3eb0b 100644 --- a/frontend/src/pages/application/index.vue +++ b/frontend/src/pages/application/index.vue @@ -31,13 +31,9 @@ diff --git a/frontend/src/stores/context.js b/frontend/src/stores/context.js index 42095144a6..65a3e54bb8 100644 --- a/frontend/src/stores/context.js +++ b/frontend/src/stores/context.js @@ -10,10 +10,12 @@ import { useProductExpertStore } from './product-expert.js' export const useContextStore = defineStore('context', { state: () => ({ route: null, + application: null, instance: null, device: null, team: null, - teamMembership: null + teamMembership: null, + isImmersive: false }), getters: { isFreeTeamType (state) { @@ -103,6 +105,8 @@ export const useContextStore = defineStore('context', { updateRoute (route) { this.route = route }, setInstance (instance) { this.instance = instance }, setDevice (device) { this.device = device }, + setApplication (application) { this.application = application }, + setIsImmersive (isImmersive) { this.isImmersive = isImmersive }, clearInstance () { this.instance = null }, setTeam (team) { this.team = team diff --git a/frontend/src/stores/product-assistant.js b/frontend/src/stores/product-assistant.js index 668751bb6f..b249f88d4f 100644 --- a/frontend/src/stores/product-assistant.js +++ b/frontend/src/stores/product-assistant.js @@ -160,8 +160,16 @@ export const useProductAssistantStore = defineStore('product-assistant', { editorState: { ...buildInitialEditorState() } }), getters: { - immersiveInstance: () => useContextStore().instance, - immersiveDevice: () => useContextStore().device, + immersiveInstance: () => { + const contextStore = useContextStore() + + return contextStore.instance && contextStore.isImmersive + }, + immersiveDevice: () => { + const contextStore = useContextStore() + + return contextStore.device && contextStore.isImmersive + }, hasUserSelection: (state) => state.selectedNodes.length > 0, hasContextSelection: (state) => state.selectedContext.length > 0, hasDebugLogsSelected () { diff --git a/frontend/src/stores/product-expert-insights-agent.js b/frontend/src/stores/product-expert-insights-agent.js index 4df5237a46..10d3c48565 100644 --- a/frontend/src/stores/product-expert-insights-agent.js +++ b/frontend/src/stores/product-expert-insights-agent.js @@ -6,6 +6,8 @@ import useTimerHelper from '../composables/TimerHelper.js' import { useContextStore } from './context.js' +import { INSIGHTS_AGENT } from './product-expert-agents.js' + export const useProductExpertInsightsAgentStore = defineStore('product-expert-insights-agent', { state: () => ({ sessionId: null, @@ -16,7 +18,8 @@ export const useProductExpertInsightsAgentStore = defineStore('product-expert-in sessionExpiredShown: false, sessionCheckTimer: null, capabilityServers: [], - selectedCapabilities: [] + selectedCapabilities: [], + mqttConnectionKey: `expert/${INSIGHTS_AGENT}` }), getters: { capabilities: (state) => state.capabilityServers.map(c => ({ diff --git a/frontend/src/stores/product-expert-support-agent.js b/frontend/src/stores/product-expert-support-agent.js index 6ee3f848a6..8d5c997036 100644 --- a/frontend/src/stores/product-expert-support-agent.js +++ b/frontend/src/stores/product-expert-support-agent.js @@ -1,6 +1,8 @@ import { defineStore } from 'pinia' import { markRaw } from 'vue' +import { SUPPORT_AGENT } from './product-expert-agents.js' + export const useProductExpertSupportAgentStore = defineStore('product-expert-support-agent', { state: () => ({ context: null, @@ -12,7 +14,8 @@ export const useProductExpertSupportAgentStore = defineStore('product-expert-sup sessionStartTime: null, sessionWarningShown: false, sessionExpiredShown: false, - sessionCheckTimer: null + sessionCheckTimer: null, + mqttConnectionKey: `expert/${SUPPORT_AGENT}` }), actions: { reset () { diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index c94a0e2840..604a05f1c8 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -3,6 +3,7 @@ import { v4 as uuidv4 } from 'uuid' import { markRaw } from 'vue' import expertApi from '../api/expert.js' +import userApi from '../api/user.js' import useTimerHelper from '../composables/TimerHelper.js' import { useAccountSettingsStore } from './account-settings.js' @@ -13,6 +14,12 @@ import { useProductExpertInsightsAgentStore } from './product-expert-insights-ag import { useProductExpertSupportAgentStore } from './product-expert-support-agent.js' import { useUxDrawersStore } from './ux-drawers.js' +import getServicesOrchestrator from '@/services/service.orchestrator' +import { useAccountAuthStore } from '@/stores/account-auth.js' + +// TODO currently hardcodded +const IS_MQTT_ENABLED = true + export const useProductExpertStore = defineStore('product-expert', { state: () => ({ agentMode: SUPPORT_AGENT, // support-agent or insights-agent @@ -42,7 +49,9 @@ export const useProductExpertStore = defineStore('product-expert', { canManagePalette () { const assistantStore = useProductAssistantStore() return !!assistantStore.immersiveInstance && !!assistantStore.supportedActions['core:manage-palette'] - } + }, + mqttConnectionKey () { return this._agentStore.mqttConnectionKey }, + sessionId () { return this._agentStore.sessionId } }, actions: { setContext ({ data, sessionId }) { @@ -193,6 +202,14 @@ export const useProductExpertStore = defineStore('product-expert', { }, sendQuery ({ query }) { + if (IS_MQTT_ENABLED && this.isSupportAgent) { + return this.handleMqttQuery({ query }) + } else { + return this.sendHttpQuery({ query }) + } + }, + + async sendHttpQuery ({ query }) { const agentStore = this._agentStore const payload = { query, @@ -211,6 +228,133 @@ export const useProductExpertStore = defineStore('product-expert', { return expertApi.chat(payload) }, + async handleMqttQuery ({ query }) { + const servicesOrchestrator = getServicesOrchestrator() + const mqttService = servicesOrchestrator.$serviceInstances.mqtt + const transactionId = uuidv4() + const sessionId = this.sessionId + const mqttConnectionKey = this.mqttConnectionKey + const authStore = useAccountAuthStore() + + if (!mqttService.hasClient(mqttConnectionKey)) await this.establishMqttComms() + const { entityId, entityType } = this._getEntityTopicPaths() + + return mqttService.publishMessage(mqttConnectionKey, { + topic: `ff/v1/expert/${authStore.user.id}/${sessionId}/${entityType}/${entityId}/support/chat/request`, + qos: 2, + payload: { + query, + context: { + ...useContextStore().expert, + agent: this.agentMode + } + }, + correlationData: transactionId, + userProperties: { + sessionId: this.sessionId + } + }) + }, + + async establishMqttComms () { + const servicesOrchestrator = getServicesOrchestrator() + const mqttService = servicesOrchestrator.$serviceInstances.mqtt + const authStore = useAccountAuthStore() + const assistantStore = useProductAssistantStore() + + await mqttService.createClient(this.mqttConnectionKey, { + getCredentials: () => userApi.initiateExpertChat({ sessionId: this.sessionId }), + onMessage: async (topic, message, packet) => { + const isChatReply = topic.endsWith('/support/chat/response') + const isToolRequest = topic.includes('/support/tool/') && topic.endsWith('/request') + if (isChatReply) { + await this.handleMessageResponse(JSON.parse(message.toString())) + } else if (isToolRequest) { + const msg = JSON.parse(message.toString()) + + const userId = authStore.user.id + const sessionId = this.sessionId + const split = topic.split('/') + const thingType = split[5] // d (device) or p (instance) + const thingId = split[6] // the id of the device or instance + const tool = split.at(-2) // the name of the tool + const responseTopic = `ff/v1/expert/${userId}/${sessionId}/${thingType}/${thingId}/support/tool/${tool}/response` + const transactionId = packet.properties?.correlationData ? new TextDecoder().decode(packet.properties.correlationData) : null + + const pushToToolCalls = (kind, message, params) => { + // push result to last message's toolCalls + const messages = this._agentStore.messages + const latestMessage = messages.at(-1) + // Initialize toolCalls array if it doesn't exist + if (!latestMessage.toolCalls2) { + latestMessage.toolCalls2 = [] + } + // Push the postMessagePayload into it + latestMessage.toolCalls2.push({ + kind, + message, + params + }) + } + + if (tool === 'expert:status-message') { + // this is just a status from the agent, we can ignore it for now, but we might want to display it in the UI later + console.log('Received status message from agent:', msg) + pushToToolCalls('status', msg.payload.status) + // just ack the msg + console.log('publishing response to', responseTopic) + await mqttService.publishMessage(this.mqttConnectionKey, { + qos: 2, + topic: responseTopic, + payload: JSON.stringify({ + ack: true + }), + correlationData: new TextEncoder().encode(transactionId), + userProperties: { sessionId } + }) + } else if (tool.startsWith('automation:')) { + // thi is an automation request, we want to execute it and return the result to the agent + const postMessagePayload = { + action: tool.replace('automation:', 'automation/'), + params: msg.payload?.params || {} + } + + // push result to last message's toolCalls + pushToToolCalls('automation', `Calling action '${postMessagePayload.action}'`, postMessagePayload.params) + + // const res = await dispatch('product/assistant/invokeAction', postMessagePayload, { root: true }) + const res = assistantStore.invokeAction(postMessagePayload) + + await mqttService.publishMessage(this.mqttConnectionKey, { + qos: 2, + topic: responseTopic, + payload: JSON.stringify({ + ack: true, + ...res + }), + correlationData: new TextEncoder().encode(transactionId), + userProperties: { sessionId } + }) + } + } + }, + onClose: () => { + // TODO add error message + }, + onConnect: () => { + const chatResponseTopic = `ff/v1/expert/${authStore.user.id}/${this._agentStore.sessionId}/+/+/support/chat/response` + mqttService.subscribe(this.mqttConnectionKey, chatResponseTopic, { qos: 2 }) + // mqttService.subscribe(this.mqttConnectionKey, toolRequestTopic, { qos: 2 }) + }, + onOffline: () => { + // TODO add error message + }, + onError: (e) => { + // TODO add error message + } + }) + }, + addWelcomeMessageIfNeeded () { const currentMode = this.agentMode const hasMessages = this._agentStore.messages.length > 0 @@ -432,6 +576,33 @@ export const useProductExpertStore = defineStore('product-expert', { } // Else: ignore messages that don't match either format }) + }, + + _getEntityTopicPaths () { + const contextStore = useContextStore() + + switch (true) { + case !!contextStore.application: + return { + entityType: 'a', + entityId: contextStore.team?.id + } + case !!contextStore.instance: + return { + entityType: 'p', + entityId: contextStore.instance.id + } + case !!contextStore.device: + return { + entityType: 'd', + entityId: contextStore.device?.id + } + default: + return { + entityType: 't', + entityId: contextStore.team?.id + } + } } }, persist: { From 3e52e265c6e62976668f1e7d4e9af0f09d60ade3 Mon Sep 17 00:00:00 2001 From: cstns Date: Fri, 17 Apr 2026 11:40:46 +0300 Subject: [PATCH 02/54] handle on connect topic subscription with wildcards and sessionId watcher to unsubscribe from stale topics and re-subscription --- frontend/src/stores/product-expert.js | 120 +++++++++++++++++++++++--- 1 file changed, 107 insertions(+), 13 deletions(-) diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 604a05f1c8..f19b61120a 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -1,6 +1,6 @@ import { defineStore } from 'pinia' import { v4 as uuidv4 } from 'uuid' -import { markRaw } from 'vue' +import { markRaw, watch } from 'vue' import expertApi from '../api/expert.js' import userApi from '../api/user.js' @@ -239,8 +239,12 @@ export const useProductExpertStore = defineStore('product-expert', { if (!mqttService.hasClient(mqttConnectionKey)) await this.establishMqttComms() const { entityId, entityType } = this._getEntityTopicPaths() + const topic = `ff/v1/expert/${authStore.user.id}/${sessionId}/${entityType}/${entityId}/support/chat/request` + + console.log('publishing to topic: ', topic) + return mqttService.publishMessage(mqttConnectionKey, { - topic: `ff/v1/expert/${authStore.user.id}/${sessionId}/${entityType}/${entityId}/support/chat/request`, + topic, qos: 2, payload: { query, @@ -262,6 +266,9 @@ export const useProductExpertStore = defineStore('product-expert', { const authStore = useAccountAuthStore() const assistantStore = useProductAssistantStore() + // todo remove, dev only + window.qwe = mqttService + await mqttService.createClient(this.mqttConnectionKey, { getCredentials: () => userApi.initiateExpertChat({ sessionId: this.sessionId }), onMessage: async (topic, message, packet) => { @@ -342,9 +349,65 @@ export const useProductExpertStore = defineStore('product-expert', { // TODO add error message }, onConnect: () => { - const chatResponseTopic = `ff/v1/expert/${authStore.user.id}/${this._agentStore.sessionId}/+/+/support/chat/response` - mqttService.subscribe(this.mqttConnectionKey, chatResponseTopic, { qos: 2 }) - // mqttService.subscribe(this.mqttConnectionKey, toolRequestTopic, { qos: 2 }) + mqttService.subscribe( + this.mqttConnectionKey, + this._topicBuilder({ + entityType: '+', + entityId: '+', + agentChannel: 'support', + topicType: 'chat', + topicAction: 'response' + }), + { qos: 2 } + ) + mqttService.subscribe( + this.mqttConnectionKey, + this._topicBuilder({ + entityType: '+', + entityId: '+', + agentChannel: 'support', + topicType: 'inflight', + topicAction: 'request' + }), + { qos: 2 } + ) + + // whenever the sessionId changes, we need to unsubscribe from previous topics and subscribe to the + // new ones based off of the new sessionId + watch( + () => this._agentStore.sessionId, + () => { + if (!mqttService.hasClient(this.mqttConnectionKey)) return + + const managedClient = mqttService.getManagedClient(this.mqttConnectionKey) + managedClient.subscriptions.forEach(subscription => { + mqttService.unsubscribe(this.mqttConnectionKey, subscription.topic) + }) + + mqttService.subscribe( + this.mqttConnectionKey, + this._topicBuilder({ + entityType: '+', + entityId: '+', + agentChannel: 'support', + topicType: 'chat', + topicAction: 'response' + }), + { qos: 2 } + ) + mqttService.subscribe( + this.mqttConnectionKey, + this._topicBuilder({ + entityType: '+', + entityId: '+', + agentChannel: 'support', + topicType: 'inflight', + topicAction: 'request' + }), + { qos: 2 } + ) + } + ) }, onOffline: () => { // TODO add error message @@ -578,31 +641,62 @@ export const useProductExpertStore = defineStore('product-expert', { }) }, - _getEntityTopicPaths () { + _getEntityTopicPaths ({ application, instance, device, team } = {}) { const contextStore = useContextStore() switch (true) { - case !!contextStore.application: + case application || !!contextStore.application: return { entityType: 'a', - entityId: contextStore.team?.id + entityId: application?.id ?? contextStore.application?.id } - case !!contextStore.instance: + case instance || !!contextStore.instance: return { entityType: 'p', - entityId: contextStore.instance.id + entityId: device?.id ?? contextStore.instance.id } - case !!contextStore.device: + case device || !!contextStore.device: return { entityType: 'd', - entityId: contextStore.device?.id + entityId: device?.id ?? contextStore.device?.id } default: return { entityType: 't', - entityId: contextStore.team?.id + entityId: team?.id ?? contextStore.team?.id } } + }, + + _topicBuilder ({ + entityType, + entityId, + agentChannel, + topicType, + topicAction + } + = { + entityType: null, + entityId: null, + agentChannel: 'support' | 'insights', + topicType: 'chat' | 'inflight', + topicAction: 'response' | 'request' + }) { + if (!entityType) throw new Error('Topic "entityType" is mandatory') + if (!entityId) throw new Error('Topic "entityId" is mandatory') + if (!['support', 'insights'].includes(agentChannel)) { + throw new Error(`"agentChannel" must be either "support" or "insights", "${agentChannel}" given`) + } + if (!['chat', 'inflight'].includes(topicType)) { + throw new Error(`"topicType" must be either "chat" or "inflight", "${topicType}" given`) + } + if (!['response', 'request'].includes(topicAction)) { + throw new Error(`"topicAction" must be either "response" or "request", "${topicAction}" given`) + } + + const authStore = useAccountAuthStore() + + return `ff/v1/expert/${authStore.user.id}/${this._agentStore.sessionId}/${entityType}/${entityId}/${agentChannel}/${topicType}/${topicAction}` } }, persist: { From a2b29969506731029f8de883bdb122798caa798b Mon Sep 17 00:00:00 2001 From: cstns Date: Fri, 17 Apr 2026 12:16:58 +0300 Subject: [PATCH 03/54] extract expert mqtt topic methods into a distinct helper --- .../services/MqttExpertTopicHelper.ts | 115 ++++++++++++++++ frontend/src/stores/product-expert.js | 129 ++++++------------ 2 files changed, 157 insertions(+), 87 deletions(-) create mode 100644 frontend/src/composables/services/MqttExpertTopicHelper.ts diff --git a/frontend/src/composables/services/MqttExpertTopicHelper.ts b/frontend/src/composables/services/MqttExpertTopicHelper.ts new file mode 100644 index 0000000000..4d300aa0d3 --- /dev/null +++ b/frontend/src/composables/services/MqttExpertTopicHelper.ts @@ -0,0 +1,115 @@ +import { useAccountAuthStore, useContextStore, useProductExpertStore } from '@/stores' + +interface EntityWithId { + id: string + [key: string]: unknown +} + +interface EntityTopicPathsOptions { + application?: EntityWithId | null + instance?: EntityWithId | null + device?: EntityWithId | null + team?: EntityWithId | null +} + +interface EntityTopicPaths { + entityType: 'a' | 'p' | 'd' | 't' + entityId: string | undefined +} + +type AgentChannel = 'support' | 'insights' +type TopicType = 'chat' | 'inflight' +type TopicAction = 'response' | 'request' + +interface DestructuredTopic { + topic: string + isReply: boolean + isInflightRequest: boolean + entityType: string + entityId: string + automation: string | null +} + +interface TopicBuilderOptions { + entityType?: EntityTopicPaths['entityType'] | null + entityId?: string | null + agentChannel?: AgentChannel + topicType?: TopicType + topicAction?: TopicAction +} + +export function useMqttExpertTopicHelper () { + function getEntityTopicPaths (options?: EntityTopicPathsOptions): EntityTopicPaths { + const { application, instance, device, team } = options ?? {} + const contextStore = useContextStore() + + switch (true) { + case !!application || !!contextStore.application: + return { + entityType: 'a', + entityId: application?.id ?? contextStore.application?.id + } + case !!instance || !!contextStore.instance: + return { + entityType: 'p', + entityId: instance?.id ?? contextStore.instance.id + } + case !!device || !!contextStore.device: + return { + entityType: 'd', + entityId: device?.id ?? contextStore.device?.id + } + default: + return { + entityType: 't', + entityId: team?.id ?? contextStore.team?.id + } + } + } + + function topicBuilder (options?: TopicBuilderOptions): string { + const { entityType, entityId, agentChannel, topicType, topicAction } = options ?? {} + + if (!entityType) throw new Error('Topic "entityType" is mandatory') + if (!entityId) throw new Error('Topic "entityId" is mandatory') + if (!agentChannel || !['support', 'insights'].includes(agentChannel)) { + throw new Error(`"agentChannel" must be either "support" or "insights", "${agentChannel}" given`) + } + if (!topicType || !['chat', 'inflight'].includes(topicType)) { + throw new Error(`"topicType" must be either "chat" or "inflight", "${topicType}" given`) + } + if (!topicAction || !['response', 'request'].includes(topicAction)) { + throw new Error(`"topicAction" must be either "response" or "request", "${topicAction}" given`) + } + + const authStore = useAccountAuthStore() + const expertStore = useProductExpertStore() + + const sessionId = expertStore.sessionId + + return `ff/v1/expert/${authStore.user.id}/${sessionId}/${entityType}/${entityId}/${agentChannel}/${topicType}/${topicAction}` + } + + function destructureTopic (topic: string): DestructuredTopic { + if (!topic || topic.length === 0) throw new Error(`Invalid topic received: "${topic}"`) + + const split = topic.split('/') + + const inflightRequest = topic.includes('/inflight/') && topic.endsWith('/request') + + return { + topic, + isReply: topic.endsWith('/response'), + isInflightRequest: inflightRequest, + entityType: split[5], + entityId: split[6], + automation: inflightRequest ? split.at(-2) ?? null : null + } + } + + return { + getEntityTopicPaths, + topicBuilder, + destructureTopic + } +} diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index f19b61120a..6c0a178918 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -14,6 +14,8 @@ import { useProductExpertInsightsAgentStore } from './product-expert-insights-ag import { useProductExpertSupportAgentStore } from './product-expert-support-agent.js' import { useUxDrawersStore } from './ux-drawers.js' +import { useMqttExpertTopicHelper } from '@/composables/services/MqttExpertTopicHelper' + import getServicesOrchestrator from '@/services/service.orchestrator' import { useAccountAuthStore } from '@/stores/account-auth.js' @@ -231,15 +233,21 @@ export const useProductExpertStore = defineStore('product-expert', { async handleMqttQuery ({ query }) { const servicesOrchestrator = getServicesOrchestrator() const mqttService = servicesOrchestrator.$serviceInstances.mqtt + const mqttTopicHelper = useMqttExpertTopicHelper() + const transactionId = uuidv4() - const sessionId = this.sessionId const mqttConnectionKey = this.mqttConnectionKey - const authStore = useAccountAuthStore() if (!mqttService.hasClient(mqttConnectionKey)) await this.establishMqttComms() - const { entityId, entityType } = this._getEntityTopicPaths() - - const topic = `ff/v1/expert/${authStore.user.id}/${sessionId}/${entityType}/${entityId}/support/chat/request` + const { entityId, entityType } = mqttTopicHelper.getEntityTopicPaths() + + const topic = mqttTopicHelper.topicBuilder({ + entityType, + entityId, + agentChannel: 'support', + topicType: 'chat', + topicAction: 'request' + }) console.log('publishing to topic: ', topic) @@ -349,9 +357,11 @@ export const useProductExpertStore = defineStore('product-expert', { // TODO add error message }, onConnect: () => { + const mqttTopicHelper = useMqttExpertTopicHelper() + mqttService.subscribe( this.mqttConnectionKey, - this._topicBuilder({ + mqttTopicHelper.topicBuilder({ entityType: '+', entityId: '+', agentChannel: 'support', @@ -360,17 +370,19 @@ export const useProductExpertStore = defineStore('product-expert', { }), { qos: 2 } ) - mqttService.subscribe( - this.mqttConnectionKey, - this._topicBuilder({ - entityType: '+', - entityId: '+', - agentChannel: 'support', - topicType: 'inflight', - topicAction: 'request' - }), - { qos: 2 } - ) + + // todo investigate why subscribing to the inflight topic fails + // mqttService.subscribe( + // this.mqttConnectionKey, + // mqttTopicHelper.topicBuilder({ + // entityType: '+', + // entityId: '+', + // agentChannel: 'support', + // topicType: 'inflight', + // topicAction: 'request' + // }), + // { qos: 2 } + // ) // whenever the sessionId changes, we need to unsubscribe from previous topics and subscribe to the // new ones based off of the new sessionId @@ -386,7 +398,7 @@ export const useProductExpertStore = defineStore('product-expert', { mqttService.subscribe( this.mqttConnectionKey, - this._topicBuilder({ + mqttTopicHelper.topicBuilder({ entityType: '+', entityId: '+', agentChannel: 'support', @@ -395,17 +407,18 @@ export const useProductExpertStore = defineStore('product-expert', { }), { qos: 2 } ) - mqttService.subscribe( - this.mqttConnectionKey, - this._topicBuilder({ - entityType: '+', - entityId: '+', - agentChannel: 'support', - topicType: 'inflight', - topicAction: 'request' - }), - { qos: 2 } - ) + // todo investigate why subscribing to the inflight topic fails + // mqttService.subscribe( + // this.mqttConnectionKey, + // mqttTopicHelper.topicBuilder({ + // entityType: '+', + // entityId: '+', + // agentChannel: 'support', + // topicType: 'inflight', + // topicAction: 'request' + // }), + // { qos: 2 } + // ) } ) }, @@ -639,64 +652,6 @@ export const useProductExpertStore = defineStore('product-expert', { } // Else: ignore messages that don't match either format }) - }, - - _getEntityTopicPaths ({ application, instance, device, team } = {}) { - const contextStore = useContextStore() - - switch (true) { - case application || !!contextStore.application: - return { - entityType: 'a', - entityId: application?.id ?? contextStore.application?.id - } - case instance || !!contextStore.instance: - return { - entityType: 'p', - entityId: device?.id ?? contextStore.instance.id - } - case device || !!contextStore.device: - return { - entityType: 'd', - entityId: device?.id ?? contextStore.device?.id - } - default: - return { - entityType: 't', - entityId: team?.id ?? contextStore.team?.id - } - } - }, - - _topicBuilder ({ - entityType, - entityId, - agentChannel, - topicType, - topicAction - } - = { - entityType: null, - entityId: null, - agentChannel: 'support' | 'insights', - topicType: 'chat' | 'inflight', - topicAction: 'response' | 'request' - }) { - if (!entityType) throw new Error('Topic "entityType" is mandatory') - if (!entityId) throw new Error('Topic "entityId" is mandatory') - if (!['support', 'insights'].includes(agentChannel)) { - throw new Error(`"agentChannel" must be either "support" or "insights", "${agentChannel}" given`) - } - if (!['chat', 'inflight'].includes(topicType)) { - throw new Error(`"topicType" must be either "chat" or "inflight", "${topicType}" given`) - } - if (!['response', 'request'].includes(topicAction)) { - throw new Error(`"topicAction" must be either "response" or "request", "${topicAction}" given`) - } - - const authStore = useAccountAuthStore() - - return `ff/v1/expert/${authStore.user.id}/${this._agentStore.sessionId}/${entityType}/${entityId}/${agentChannel}/${topicType}/${topicAction}` } }, persist: { From 3650a81a1f7407097ea8e0703ea5782486943956 Mon Sep 17 00:00:00 2001 From: cstns Date: Fri, 17 Apr 2026 13:00:17 +0300 Subject: [PATCH 04/54] renale automation to something more explicit --- .../services/MqttExpertTopicHelper.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/frontend/src/composables/services/MqttExpertTopicHelper.ts b/frontend/src/composables/services/MqttExpertTopicHelper.ts index 4d300aa0d3..e172b86e44 100644 --- a/frontend/src/composables/services/MqttExpertTopicHelper.ts +++ b/frontend/src/composables/services/MqttExpertTopicHelper.ts @@ -27,7 +27,7 @@ interface DestructuredTopic { isInflightRequest: boolean entityType: string entityId: string - automation: string | null + tool: string | null } interface TopicBuilderOptions { @@ -87,7 +87,18 @@ export function useMqttExpertTopicHelper () { const sessionId = expertStore.sessionId - return `ff/v1/expert/${authStore.user.id}/${sessionId}/${entityType}/${entityId}/${agentChannel}/${topicType}/${topicAction}` + return [ + 'ff', + 'v1', + 'expert', + authStore.user.id, + sessionId, + entityType, + entityId, + agentChannel, + topicType, + topicAction + ].join('/') } function destructureTopic (topic: string): DestructuredTopic { @@ -103,7 +114,7 @@ export function useMqttExpertTopicHelper () { isInflightRequest: inflightRequest, entityType: split[5], entityId: split[6], - automation: inflightRequest ? split.at(-2) ?? null : null + tool: inflightRequest ? split.at(-2) ?? null : null } } From 9c9388a60df5d51aadb3a0bf8c6818b3857f6435 Mon Sep 17 00:00:00 2001 From: cstns Date: Fri, 17 Apr 2026 13:00:17 +0300 Subject: [PATCH 05/54] rename automation to something more explicit --- .../services/MqttExpertTopicHelper.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/frontend/src/composables/services/MqttExpertTopicHelper.ts b/frontend/src/composables/services/MqttExpertTopicHelper.ts index 4d300aa0d3..e172b86e44 100644 --- a/frontend/src/composables/services/MqttExpertTopicHelper.ts +++ b/frontend/src/composables/services/MqttExpertTopicHelper.ts @@ -27,7 +27,7 @@ interface DestructuredTopic { isInflightRequest: boolean entityType: string entityId: string - automation: string | null + tool: string | null } interface TopicBuilderOptions { @@ -87,7 +87,18 @@ export function useMqttExpertTopicHelper () { const sessionId = expertStore.sessionId - return `ff/v1/expert/${authStore.user.id}/${sessionId}/${entityType}/${entityId}/${agentChannel}/${topicType}/${topicAction}` + return [ + 'ff', + 'v1', + 'expert', + authStore.user.id, + sessionId, + entityType, + entityId, + agentChannel, + topicType, + topicAction + ].join('/') } function destructureTopic (topic: string): DestructuredTopic { @@ -103,7 +114,7 @@ export function useMqttExpertTopicHelper () { isInflightRequest: inflightRequest, entityType: split[5], entityId: split[6], - automation: inflightRequest ? split.at(-2) ?? null : null + tool: inflightRequest ? split.at(-2) ?? null : null } } From 51c0aeda8d5aa6387073bf34bde7ec17b1b3506d Mon Sep 17 00:00:00 2001 From: cstns Date: Fri, 17 Apr 2026 14:11:20 +0300 Subject: [PATCH 06/54] store inflight request to an agent payload as a resolver to track pending messages and tool calls, handle assistant replies based on toolCall --- .../services/MqttExpertTopicHelper.ts | 2 + frontend/src/stores/product-assistant.js | 10 + .../stores/product-expert-insights-agent.js | 3 +- .../stores/product-expert-support-agent.js | 3 +- frontend/src/stores/product-expert.js | 313 +++++++++--------- 5 files changed, 173 insertions(+), 158 deletions(-) diff --git a/frontend/src/composables/services/MqttExpertTopicHelper.ts b/frontend/src/composables/services/MqttExpertTopicHelper.ts index e172b86e44..826c7f7497 100644 --- a/frontend/src/composables/services/MqttExpertTopicHelper.ts +++ b/frontend/src/composables/services/MqttExpertTopicHelper.ts @@ -67,6 +67,7 @@ export function useMqttExpertTopicHelper () { } } + // todo rename to buildTopic function topicBuilder (options?: TopicBuilderOptions): string { const { entityType, entityId, agentChannel, topicType, topicAction } = options ?? {} @@ -101,6 +102,7 @@ export function useMqttExpertTopicHelper () { ].join('/') } + // todo rename to parseTopic function destructureTopic (topic: string): DestructuredTopic { if (!topic || topic.length === 0) throw new Error(`Invalid topic received: "${topic}"`) diff --git a/frontend/src/stores/product-assistant.js b/frontend/src/stores/product-assistant.js index b249f88d4f..a2b3e386fa 100644 --- a/frontend/src/stores/product-assistant.js +++ b/frontend/src/stores/product-assistant.js @@ -3,6 +3,7 @@ import SemVer from 'semver' import messagingService from '@/services/post-message.service' import { useContextStore } from '@/stores/context.js' +import { useProductExpertStore } from '@/stores/product-expert.js' const MAX_DEBUG_LOG_ENTRIES = 100 // maximum number of debug log entries to keep @@ -268,6 +269,15 @@ export const useProductAssistantStore = defineStore('product-assistant', { return } + // todo define how we receive the transaction id back + if (payload.data.transactionId) { + const expertStore = useProductExpertStore() + await expertStore.handleAgentReply({ + transactionId: payload.data.transactionId, + response: payload + }) + } + switch (true) { case payload.data.type === 'assistant-ready': this.version = payload.data.version diff --git a/frontend/src/stores/product-expert-insights-agent.js b/frontend/src/stores/product-expert-insights-agent.js index 10d3c48565..79b9ec7f6c 100644 --- a/frontend/src/stores/product-expert-insights-agent.js +++ b/frontend/src/stores/product-expert-insights-agent.js @@ -19,7 +19,8 @@ export const useProductExpertInsightsAgentStore = defineStore('product-expert-in sessionCheckTimer: null, capabilityServers: [], selectedCapabilities: [], - mqttConnectionKey: `expert/${INSIGHTS_AGENT}` + mqttConnectionKey: `expert/${INSIGHTS_AGENT}`, + inFlightRequests: new Map() }), getters: { capabilities: (state) => state.capabilityServers.map(c => ({ diff --git a/frontend/src/stores/product-expert-support-agent.js b/frontend/src/stores/product-expert-support-agent.js index 8d5c997036..e41aa6acd5 100644 --- a/frontend/src/stores/product-expert-support-agent.js +++ b/frontend/src/stores/product-expert-support-agent.js @@ -15,7 +15,8 @@ export const useProductExpertSupportAgentStore = defineStore('product-expert-sup sessionWarningShown: false, sessionExpiredShown: false, sessionCheckTimer: null, - mqttConnectionKey: `expert/${SUPPORT_AGENT}` + mqttConnectionKey: `expert/${SUPPORT_AGENT}`, + inFlightRequests: new Map() }), actions: { reset () { diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 6c0a178918..df21f1a877 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -17,7 +17,6 @@ import { useUxDrawersStore } from './ux-drawers.js' import { useMqttExpertTopicHelper } from '@/composables/services/MqttExpertTopicHelper' import getServicesOrchestrator from '@/services/service.orchestrator' -import { useAccountAuthStore } from '@/stores/account-auth.js' // TODO currently hardcodded const IS_MQTT_ENABLED = true @@ -34,6 +33,11 @@ export const useProductExpertStore = defineStore('product-expert', { ? useProductExpertSupportAgentStore() : useProductExpertInsightsAgentStore() }, + _inFlightRequests () { + return this.agentMode === SUPPORT_AGENT + ? useProductExpertSupportAgentStore().inFlightRequests + : useProductExpertInsightsAgentStore().inFlightRequests + }, abortController () { return this._agentStore.abortController }, messages () { return this._agentStore.messages }, hasMessages () { return this._agentStore.messages.length > 0 }, @@ -102,40 +106,6 @@ export const useProductExpertStore = defineStore('product-expert', { }) }) }, - - openAssistantDrawer (options = {}) { - const featuresCheck = useAccountSettingsStore().featuresCheck - if (featuresCheck.isExpertAssistantFeatureEnabled === false) return - - useProductExpertInsightsAgentStore().getCapabilities() - // Lazy import to avoid circular dep: product-expert.js → ExpertDrawer.vue → product-expert.js - return import('../components/drawers/expert/ExpertDrawer.vue') - .then(({ default: ExpertDrawer }) => useUxDrawersStore().openRightDrawer({ - component: markRaw(ExpertDrawer), - fixed: options?.openPinned === true, - closeOnClickOutside: options?.openPinned !== true - })) - }, - - wakeUpAssistant ({ shouldHydrateMessages = false } = {}) { - if (this.shouldWakeUpAssistant) { - const featuresCheck = useAccountSettingsStore().featuresCheck - if (featuresCheck.isExpertAssistantFeatureEnabled === false) return - - this.clearWakeUp() - - if (shouldHydrateMessages) { - this.hydrateMessages(this._agentStore.context) - } - - this.loadingVariant = 'transfer' - - return this.openAssistantDrawer({ openPinned: useUxDrawersStore().rightDrawer.expertState.pinned }) - .then(() => this.hydrateClient()) - .then(() => { this.loadingVariant = this.agentMode }) - } - }, - async handleQuery ({ query }) { const agentStore = this._agentStore @@ -169,40 +139,6 @@ export const useProductExpertStore = defineStore('product-expert', { agentStore.abortController = null } }, - - async handleMessageResponse (response) { - if (response.answer && Array.isArray(response.answer)) { - this.addAiMessage(response) - } - }, - - async startOver () { - const agentStore = this._agentStore - agentStore.sessionId = uuidv4() - agentStore.messages = [] - - // Reset session timing - this.resetSessionTimer() - this.startSessionTimer() - - // Clear resource selection - const insightsStore = useProductExpertInsightsAgentStore() - insightsStore.setSelectedCapabilities([]) - await insightsStore.getCapabilities() - - // Add welcome message for current mode - this.addWelcomeMessageIfNeeded() - }, - - setAbortController (controller) { - this._agentStore.abortController = controller ? markRaw(controller) : null - }, - - reset () { - this._agentStore.reset() - this.$reset() - }, - sendQuery ({ query }) { if (IS_MQTT_ENABLED && this.isSupportAgent) { return this.handleMqttQuery({ query }) @@ -210,7 +146,6 @@ export const useProductExpertStore = defineStore('product-expert', { return this.sendHttpQuery({ query }) } }, - async sendHttpQuery ({ query }) { const agentStore = this._agentStore const payload = { @@ -229,7 +164,6 @@ export const useProductExpertStore = defineStore('product-expert', { return expertApi.chat(payload) }, - async handleMqttQuery ({ query }) { const servicesOrchestrator = getServicesOrchestrator() const mqttService = servicesOrchestrator.$serviceInstances.mqtt @@ -238,6 +172,9 @@ export const useProductExpertStore = defineStore('product-expert', { const transactionId = uuidv4() const mqttConnectionKey = this.mqttConnectionKey + // add the query as an inFlight request + this._inFlightRequests.set(transactionId, { query }) + if (!mqttService.hasClient(mqttConnectionKey)) await this.establishMqttComms() const { entityId, entityType } = mqttTopicHelper.getEntityTopicPaths() @@ -249,8 +186,6 @@ export const useProductExpertStore = defineStore('product-expert', { topicAction: 'request' }) - console.log('publishing to topic: ', topic) - return mqttService.publishMessage(mqttConnectionKey, { topic, qos: 2, @@ -267,12 +202,10 @@ export const useProductExpertStore = defineStore('product-expert', { } }) }, - async establishMqttComms () { const servicesOrchestrator = getServicesOrchestrator() const mqttService = servicesOrchestrator.$serviceInstances.mqtt - const authStore = useAccountAuthStore() - const assistantStore = useProductAssistantStore() + const topicHelper = useMqttExpertTopicHelper() // todo remove, dev only window.qwe = mqttService @@ -280,77 +213,22 @@ export const useProductExpertStore = defineStore('product-expert', { await mqttService.createClient(this.mqttConnectionKey, { getCredentials: () => userApi.initiateExpertChat({ sessionId: this.sessionId }), onMessage: async (topic, message, packet) => { - const isChatReply = topic.endsWith('/support/chat/response') - const isToolRequest = topic.includes('/support/tool/') && topic.endsWith('/request') - if (isChatReply) { - await this.handleMessageResponse(JSON.parse(message.toString())) - } else if (isToolRequest) { - const msg = JSON.parse(message.toString()) - - const userId = authStore.user.id - const sessionId = this.sessionId - const split = topic.split('/') - const thingType = split[5] // d (device) or p (instance) - const thingId = split[6] // the id of the device or instance - const tool = split.at(-2) // the name of the tool - const responseTopic = `ff/v1/expert/${userId}/${sessionId}/${thingType}/${thingId}/support/tool/${tool}/response` - const transactionId = packet.properties?.correlationData ? new TextDecoder().decode(packet.properties.correlationData) : null - - const pushToToolCalls = (kind, message, params) => { - // push result to last message's toolCalls - const messages = this._agentStore.messages - const latestMessage = messages.at(-1) - // Initialize toolCalls array if it doesn't exist - if (!latestMessage.toolCalls2) { - latestMessage.toolCalls2 = [] - } - // Push the postMessagePayload into it - latestMessage.toolCalls2.push({ - kind, - message, - params - }) - } + const parsedTopic = topicHelper.destructureTopic(topic) + const transactionId = packet.properties?.correlationData ? new TextDecoder().decode(packet.properties.correlationData) : null - if (tool === 'expert:status-message') { - // this is just a status from the agent, we can ignore it for now, but we might want to display it in the UI later - console.log('Received status message from agent:', msg) - pushToToolCalls('status', msg.payload.status) - // just ack the msg - console.log('publishing response to', responseTopic) - await mqttService.publishMessage(this.mqttConnectionKey, { - qos: 2, - topic: responseTopic, - payload: JSON.stringify({ - ack: true - }), - correlationData: new TextEncoder().encode(transactionId), - userProperties: { sessionId } - }) - } else if (tool.startsWith('automation:')) { - // thi is an automation request, we want to execute it and return the result to the agent - const postMessagePayload = { - action: tool.replace('automation:', 'automation/'), - params: msg.payload?.params || {} - } - - // push result to last message's toolCalls - pushToToolCalls('automation', `Calling action '${postMessagePayload.action}'`, postMessagePayload.params) - - // const res = await dispatch('product/assistant/invokeAction', postMessagePayload, { root: true }) - const res = assistantStore.invokeAction(postMessagePayload) - - await mqttService.publishMessage(this.mqttConnectionKey, { - qos: 2, - topic: responseTopic, - payload: JSON.stringify({ - ack: true, - ...res - }), - correlationData: new TextEncoder().encode(transactionId), - userProperties: { sessionId } - }) - } + if (parsedTopic.isReply) { + // remove inFlight request because it is now resolved + this._inFlightRequests.delete(transactionId) + + // handle the response + await this.handleMessageResponse(JSON.parse(message.toString())) + } else if (parsedTopic.isInflightRequest) { + await this.handleInFlightRequest({ + topic, + message, + packet, + transactionId + }) } }, onClose: () => { @@ -430,7 +308,140 @@ export const useProductExpertStore = defineStore('product-expert', { } }) }, + async handleInFlightRequest ({ topic, message, packet, transactionId }) { + const servicesOrchestrator = getServicesOrchestrator() + const mqttService = servicesOrchestrator.$serviceInstances.mqtt + + const assistantStore = useProductAssistantStore() + + const topicHelper = useMqttExpertTopicHelper() + const parsedTopic = topicHelper.destructureTopic(topic) + + const msg = JSON.parse(message.toString()) + const sessionId = this.sessionId + + if (parsedTopic.tool === 'expert:status-message') { + const responseTopic = topicHelper.topicBuilder({ + entityType: parsedTopic.entityType, + entityId: parsedTopic.entityId, + agentChannel: 'support', + topicType: 'inflight', + topicAction: 'response' + }) + + // this is just a status from the agent, we can ignore it for now, but we might want to display it in the UI later + // just ack the msg + // todo do we need it? + await mqttService.publishMessage(this.mqttConnectionKey, { + qos: 2, + topic: responseTopic, + payload: JSON.stringify({ + ack: true + }), + correlationData: new TextEncoder().encode(transactionId), + userProperties: { sessionId } + }) + } else if (parsedTopic.tool.startsWith('automation:')) { + // thi is an automation request, we want to execute it and return the result to the agent + const postMessagePayload = { + action: parsedTopic.tool.replace('automation:', 'automation/'), + params: msg.payload?.params || {}, + userProperties: { sessionId }, + transactionId + } + + // adding the inFlightRequest to the queue because the message is passed along to the nr-assistant + this._inFlightRequests.set(transactionId, { topic, message, packet, postMessagePayload }) + + await assistantStore.invokeAction(postMessagePayload) + } + }, + async handleAgentReply ({ transactionId, response }) { + if (!this._inFlightRequests.has(transactionId)) return false + + const servicesOrchestrator = getServicesOrchestrator() + const mqttService = servicesOrchestrator.$serviceInstances.mqtt + const topicHelper = useMqttExpertTopicHelper() + + const originalTopic = topicHelper.destructureTopic() + const replyTopic = topicHelper.topicBuilder({ + entityType: originalTopic.entityType, + entityId: originalTopic.entityId, + agentChannel: 'support', + topicType: 'inflight', + topicAction: 'response' + }) + + await mqttService.publishMessage(this.mqttConnectionKey, { + qos: 2, + topic: replyTopic, + payload: JSON.stringify(response), + correlationData: new TextEncoder().encode(transactionId), + userProperties: { sessionId: this.sessionId } + }) + + this._inFlightRequests.delete(transactionId) + }, + async handleMessageResponse (response) { + if (response.answer && Array.isArray(response.answer)) { + this.addAiMessage(response) + } + }, + async startOver () { + const agentStore = this._agentStore + agentStore.sessionId = uuidv4() + agentStore.messages = [] + + // Reset session timing + this.resetSessionTimer() + this.startSessionTimer() + // Clear resource selection + const insightsStore = useProductExpertInsightsAgentStore() + insightsStore.setSelectedCapabilities([]) + await insightsStore.getCapabilities() + + // Add welcome message for current mode + this.addWelcomeMessageIfNeeded() + }, + openAssistantDrawer (options = {}) { + const featuresCheck = useAccountSettingsStore().featuresCheck + if (featuresCheck.isExpertAssistantFeatureEnabled === false) return + + useProductExpertInsightsAgentStore().getCapabilities() + // Lazy import to avoid circular dep: product-expert.js → ExpertDrawer.vue → product-expert.js + return import('../components/drawers/expert/ExpertDrawer.vue') + .then(({ default: ExpertDrawer }) => useUxDrawersStore().openRightDrawer({ + component: markRaw(ExpertDrawer), + fixed: options?.openPinned === true, + closeOnClickOutside: options?.openPinned !== true + })) + }, + wakeUpAssistant ({ shouldHydrateMessages = false } = {}) { + if (this.shouldWakeUpAssistant) { + const featuresCheck = useAccountSettingsStore().featuresCheck + if (featuresCheck.isExpertAssistantFeatureEnabled === false) return + + this.clearWakeUp() + + if (shouldHydrateMessages) { + this.hydrateMessages(this._agentStore.context) + } + + this.loadingVariant = 'transfer' + + return this.openAssistantDrawer({ openPinned: useUxDrawersStore().rightDrawer.expertState.pinned }) + .then(() => this.hydrateClient()) + .then(() => { this.loadingVariant = this.agentMode }) + } + }, + setAbortController (controller) { + this._agentStore.abortController = controller ? markRaw(controller) : null + }, + reset () { + this._agentStore.reset() + this.$reset() + }, addWelcomeMessageIfNeeded () { const currentMode = this.agentMode const hasMessages = this._agentStore.messages.length > 0 @@ -461,7 +472,6 @@ export const useProductExpertStore = defineStore('product-expert', { this.addPredefinedAiMessage(message) } }, - // Session timing actions startSessionTimer () { const agentStore = this._agentStore @@ -503,7 +513,6 @@ export const useProductExpertStore = defineStore('product-expert', { agentStore.setSessionCheckTimer(timer) }, - resetSessionTimer () { const agentStore = this._agentStore if (agentStore.sessionCheckTimer) { @@ -514,7 +523,6 @@ export const useProductExpertStore = defineStore('product-expert', { agentStore.sessionWarningShown = false agentStore.sessionExpiredShown = false }, - /** * * @param {'support-agent' | 'insights-agent'} mode @@ -524,7 +532,6 @@ export const useProductExpertStore = defineStore('product-expert', { this.agentMode = mode this.loadingVariant = mode }, - /** * Adds a system message to the application's message store. * @@ -542,7 +549,6 @@ export const useProductExpertStore = defineStore('product-expert', { _uuid: uuidv4() }) }, - addPredefinedAiMessage (message) { this._agentStore.messages.push({ _type: 'ai', @@ -552,7 +558,6 @@ export const useProductExpertStore = defineStore('product-expert', { _uuid: uuidv4() }) }, - addAiMessage (message) { const answer = message.answer ? message.answer.map(a => ({ ...a, _uuid: uuidv4(), _streamed: false })) @@ -566,7 +571,6 @@ export const useProductExpertStore = defineStore('product-expert', { _uuid: uuidv4() }) }, - addUserMessage (message) { this._agentStore.messages.push({ _type: 'human', @@ -575,7 +579,6 @@ export const useProductExpertStore = defineStore('product-expert', { _uuid: uuidv4() }) }, - updateMessageStreamedState (uuid) { let message = useProductExpertSupportAgentStore().messages.find(m => m._uuid === uuid) if (!message) { @@ -585,7 +588,6 @@ export const useProductExpertStore = defineStore('product-expert', { message._streamed = true } }, - updateAnswerStreamedState ({ messageUuid, answerUuid }) { let message = useProductExpertSupportAgentStore().messages.find(m => m._uuid === messageUuid) if (!message) { @@ -598,7 +600,6 @@ export const useProductExpertStore = defineStore('product-expert', { } } }, - hydrateMessages (messages) { if (!messages) return messages.forEach((message) => { From 3b6bea8eedaf757798eee5717dc7859756b99c4a Mon Sep 17 00:00:00 2001 From: cstns Date: Fri, 17 Apr 2026 14:14:15 +0300 Subject: [PATCH 07/54] rename topic helper methods --- .../services/MqttExpertTopicHelper.ts | 14 ++++++------- frontend/src/stores/product-expert.js | 20 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/frontend/src/composables/services/MqttExpertTopicHelper.ts b/frontend/src/composables/services/MqttExpertTopicHelper.ts index 826c7f7497..2d7f246bd9 100644 --- a/frontend/src/composables/services/MqttExpertTopicHelper.ts +++ b/frontend/src/composables/services/MqttExpertTopicHelper.ts @@ -21,7 +21,7 @@ type AgentChannel = 'support' | 'insights' type TopicType = 'chat' | 'inflight' type TopicAction = 'response' | 'request' -interface DestructuredTopic { +interface ParsedTopic { topic: string isReply: boolean isInflightRequest: boolean @@ -30,7 +30,7 @@ interface DestructuredTopic { tool: string | null } -interface TopicBuilderOptions { +interface BuildTopicOptions { entityType?: EntityTopicPaths['entityType'] | null entityId?: string | null agentChannel?: AgentChannel @@ -67,8 +67,7 @@ export function useMqttExpertTopicHelper () { } } - // todo rename to buildTopic - function topicBuilder (options?: TopicBuilderOptions): string { + function buildTopic (options?: BuildTopicOptions): string { const { entityType, entityId, agentChannel, topicType, topicAction } = options ?? {} if (!entityType) throw new Error('Topic "entityType" is mandatory') @@ -102,8 +101,7 @@ export function useMqttExpertTopicHelper () { ].join('/') } - // todo rename to parseTopic - function destructureTopic (topic: string): DestructuredTopic { + function parseTopic (topic: string): ParsedTopic { if (!topic || topic.length === 0) throw new Error(`Invalid topic received: "${topic}"`) const split = topic.split('/') @@ -122,7 +120,7 @@ export function useMqttExpertTopicHelper () { return { getEntityTopicPaths, - topicBuilder, - destructureTopic + buildTopic, + parseTopic } } diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index df21f1a877..6c875143f6 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -178,7 +178,7 @@ export const useProductExpertStore = defineStore('product-expert', { if (!mqttService.hasClient(mqttConnectionKey)) await this.establishMqttComms() const { entityId, entityType } = mqttTopicHelper.getEntityTopicPaths() - const topic = mqttTopicHelper.topicBuilder({ + const topic = mqttTopicHelper.buildTopic({ entityType, entityId, agentChannel: 'support', @@ -213,7 +213,7 @@ export const useProductExpertStore = defineStore('product-expert', { await mqttService.createClient(this.mqttConnectionKey, { getCredentials: () => userApi.initiateExpertChat({ sessionId: this.sessionId }), onMessage: async (topic, message, packet) => { - const parsedTopic = topicHelper.destructureTopic(topic) + const parsedTopic = topicHelper.parseTopic(topic) const transactionId = packet.properties?.correlationData ? new TextDecoder().decode(packet.properties.correlationData) : null if (parsedTopic.isReply) { @@ -239,7 +239,7 @@ export const useProductExpertStore = defineStore('product-expert', { mqttService.subscribe( this.mqttConnectionKey, - mqttTopicHelper.topicBuilder({ + mqttTopicHelper.buildTopic({ entityType: '+', entityId: '+', agentChannel: 'support', @@ -252,7 +252,7 @@ export const useProductExpertStore = defineStore('product-expert', { // todo investigate why subscribing to the inflight topic fails // mqttService.subscribe( // this.mqttConnectionKey, - // mqttTopicHelper.topicBuilder({ + // mqttTopicHelper.buildTopic({ // entityType: '+', // entityId: '+', // agentChannel: 'support', @@ -276,7 +276,7 @@ export const useProductExpertStore = defineStore('product-expert', { mqttService.subscribe( this.mqttConnectionKey, - mqttTopicHelper.topicBuilder({ + mqttTopicHelper.buildTopic({ entityType: '+', entityId: '+', agentChannel: 'support', @@ -288,7 +288,7 @@ export const useProductExpertStore = defineStore('product-expert', { // todo investigate why subscribing to the inflight topic fails // mqttService.subscribe( // this.mqttConnectionKey, - // mqttTopicHelper.topicBuilder({ + // mqttTopicHelper.buildTopic({ // entityType: '+', // entityId: '+', // agentChannel: 'support', @@ -315,13 +315,13 @@ export const useProductExpertStore = defineStore('product-expert', { const assistantStore = useProductAssistantStore() const topicHelper = useMqttExpertTopicHelper() - const parsedTopic = topicHelper.destructureTopic(topic) + const parsedTopic = topicHelper.parseTopic(topic) const msg = JSON.parse(message.toString()) const sessionId = this.sessionId if (parsedTopic.tool === 'expert:status-message') { - const responseTopic = topicHelper.topicBuilder({ + const responseTopic = topicHelper.buildTopic({ entityType: parsedTopic.entityType, entityId: parsedTopic.entityId, agentChannel: 'support', @@ -363,8 +363,8 @@ export const useProductExpertStore = defineStore('product-expert', { const mqttService = servicesOrchestrator.$serviceInstances.mqtt const topicHelper = useMqttExpertTopicHelper() - const originalTopic = topicHelper.destructureTopic() - const replyTopic = topicHelper.topicBuilder({ + const originalTopic = topicHelper.parseTopic() + const replyTopic = topicHelper.buildTopic({ entityType: originalTopic.entityType, entityId: originalTopic.entityId, agentChannel: 'support', From ad2e386b578a8c2b82b72b1afb9895ad311e975a Mon Sep 17 00:00:00 2001 From: cstns Date: Fri, 17 Apr 2026 14:30:24 +0300 Subject: [PATCH 08/54] extend parsed topic and add example topics --- .../src/composables/services/MqttExpertTopicHelper.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/frontend/src/composables/services/MqttExpertTopicHelper.ts b/frontend/src/composables/services/MqttExpertTopicHelper.ts index 2d7f246bd9..7a34306138 100644 --- a/frontend/src/composables/services/MqttExpertTopicHelper.ts +++ b/frontend/src/composables/services/MqttExpertTopicHelper.ts @@ -27,6 +27,9 @@ interface ParsedTopic { isInflightRequest: boolean entityType: string entityId: string + agentChannel: 'support' | 'insights' | string + topicType: string | 'chat' | 'inflight' | null + topicAction: string | 'request' | 'response' | null tool: string | null } @@ -102,18 +105,24 @@ export function useMqttExpertTopicHelper () { } function parseTopic (topic: string): ParsedTopic { + // topic examples + // ff/v1/expert/////// + // ff/v1/expert//////// + if (!topic || topic.length === 0) throw new Error(`Invalid topic received: "${topic}"`) const split = topic.split('/') const inflightRequest = topic.includes('/inflight/') && topic.endsWith('/request') - return { topic, isReply: topic.endsWith('/response'), isInflightRequest: inflightRequest, entityType: split[5], entityId: split[6], + agentChannel: split[7], + topicType: split[8], + topicAction: split.at(-1), tool: inflightRequest ? split.at(-2) ?? null : null } } From 3668bcd8b1a3b169f4accba12794a95820777b07 Mon Sep 17 00:00:00 2001 From: cstns Date: Fri, 17 Apr 2026 18:53:25 +0300 Subject: [PATCH 09/54] fix missing immersive entity id when sending postMessages to nr-asistant --- frontend/src/stores/product-assistant.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/stores/product-assistant.js b/frontend/src/stores/product-assistant.js index 6a927872ba..aa9661ed1f 100644 --- a/frontend/src/stores/product-assistant.js +++ b/frontend/src/stores/product-assistant.js @@ -496,6 +496,7 @@ export const useProductAssistantStore = defineStore('product-assistant', { }, sendMessage (payload) { const orchestrator = getServicesOrchestrator() + const contextStore = useContextStore() orchestrator.$serviceInstances.postMessage.sendMessage({ message: { @@ -503,7 +504,7 @@ export const useProductAssistantStore = defineStore('product-assistant', { ...this.scope }, target: window.frames['immersive-editor-iframe'], - targetOrigin: this.immersiveInstance?.url + targetOrigin: (contextStore.instance || contextStore.device)?.url }) } } From 05bc0c3a819f24fde9c966af8226a3b3edb2e9e1 Mon Sep 17 00:00:00 2001 From: cstns Date: Fri, 17 Apr 2026 22:09:38 +0300 Subject: [PATCH 10/54] fix subscribing to inflightTopics, handle reconnect done but with a big asterisk when re-creating the mqtt client and fixes to the topic builder --- .../services/MqttExpertTopicHelper.ts | 13 +-- frontend/src/stores/product-assistant.js | 6 +- frontend/src/stores/product-expert.js | 90 +++++++++++-------- 3 files changed, 66 insertions(+), 43 deletions(-) diff --git a/frontend/src/composables/services/MqttExpertTopicHelper.ts b/frontend/src/composables/services/MqttExpertTopicHelper.ts index 7a34306138..cb5a71b6ad 100644 --- a/frontend/src/composables/services/MqttExpertTopicHelper.ts +++ b/frontend/src/composables/services/MqttExpertTopicHelper.ts @@ -30,7 +30,7 @@ interface ParsedTopic { agentChannel: 'support' | 'insights' | string topicType: string | 'chat' | 'inflight' | null topicAction: string | 'request' | 'response' | null - tool: string | null + inflightType: string | null } interface BuildTopicOptions { @@ -38,7 +38,8 @@ interface BuildTopicOptions { entityId?: string | null agentChannel?: AgentChannel topicType?: TopicType - topicAction?: TopicAction + topicAction?: TopicAction, + inflightType?: string | null } export function useMqttExpertTopicHelper () { @@ -71,7 +72,7 @@ export function useMqttExpertTopicHelper () { } function buildTopic (options?: BuildTopicOptions): string { - const { entityType, entityId, agentChannel, topicType, topicAction } = options ?? {} + const { entityType, entityId, agentChannel, topicType, topicAction, inflightType } = options ?? {} if (!entityType) throw new Error('Topic "entityType" is mandatory') if (!entityId) throw new Error('Topic "entityId" is mandatory') @@ -100,8 +101,10 @@ export function useMqttExpertTopicHelper () { entityId, agentChannel, topicType, + inflightType, topicAction - ].join('/') + ] + .filter(str => str).join('/') } function parseTopic (topic: string): ParsedTopic { @@ -123,7 +126,7 @@ export function useMqttExpertTopicHelper () { agentChannel: split[7], topicType: split[8], topicAction: split.at(-1), - tool: inflightRequest ? split.at(-2) ?? null : null + inflightType: inflightRequest ? split.at(-2) ?? null : null } } diff --git a/frontend/src/stores/product-assistant.js b/frontend/src/stores/product-assistant.js index aa9661ed1f..913bf4438b 100644 --- a/frontend/src/stores/product-assistant.js +++ b/frontend/src/stores/product-assistant.js @@ -458,11 +458,13 @@ export const useProductAssistantStore = defineStore('product-assistant', { resetContextSelection () { this.selectedContext = this.availableContextOptions }, - async invokeAction ({ action, params }) { + async invokeAction ({ action, params, userProperties, transactionId }) { return this.sendMessage({ type: 'invoke-action', action, - params + params, + userProperties, + transactionId }) }, async sendFlowsToImport (flowsJson) { diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 6c875143f6..4ce17a48e1 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -249,32 +249,45 @@ export const useProductExpertStore = defineStore('product-expert', { { qos: 2 } ) - // todo investigate why subscribing to the inflight topic fails - // mqttService.subscribe( - // this.mqttConnectionKey, - // mqttTopicHelper.buildTopic({ - // entityType: '+', - // entityId: '+', - // agentChannel: 'support', - // topicType: 'inflight', - // topicAction: 'request' - // }), - // { qos: 2 } - // ) + mqttService.subscribe( + this.mqttConnectionKey, + mqttTopicHelper.buildTopic({ + entityType: '+', + entityId: '+', + agentChannel: 'support', + topicType: 'inflight', + topicAction: 'request', + inflightType: '+' + }), + { qos: 2 } + ) // whenever the sessionId changes, we need to unsubscribe from previous topics and subscribe to the // new ones based off of the new sessionId watch( () => this._agentStore.sessionId, - () => { + async () => { if (!mqttService.hasClient(this.mqttConnectionKey)) return + const timerHelper = useTimerHelper() + await mqttService.destroyClient(this.mqttConnectionKey) + + // todo extract all hooks into atomic methods and prevent this recursion from happening + // good enough for demo purposes + await this.establishMqttComms() + // todo also, getting required creds fails from time to time because we're creating the client + // before the backend successfully remove the old one + const managedClient = mqttService.getManagedClient(this.mqttConnectionKey) - managedClient.subscriptions.forEach(subscription => { - mqttService.unsubscribe(this.mqttConnectionKey, subscription.topic) - }) - mqttService.subscribe( + await timerHelper.waitWhile( + () => ['connected'].includes(managedClient.status), + { intervalMs: 200, cutoffTries: 50 } + ) + + await new Promise(resolve => setTimeout(resolve, 5000)) + + await mqttService.subscribe( this.mqttConnectionKey, mqttTopicHelper.buildTopic({ entityType: '+', @@ -285,18 +298,18 @@ export const useProductExpertStore = defineStore('product-expert', { }), { qos: 2 } ) - // todo investigate why subscribing to the inflight topic fails - // mqttService.subscribe( - // this.mqttConnectionKey, - // mqttTopicHelper.buildTopic({ - // entityType: '+', - // entityId: '+', - // agentChannel: 'support', - // topicType: 'inflight', - // topicAction: 'request' - // }), - // { qos: 2 } - // ) + await mqttService.subscribe( + this.mqttConnectionKey, + mqttTopicHelper.buildTopic({ + entityType: '+', + entityId: '+', + agentChannel: 'support', + topicType: 'inflight', + topicAction: 'request', + inflightType: '+' + }), + { qos: 2 } + ) } ) }, @@ -320,13 +333,14 @@ export const useProductExpertStore = defineStore('product-expert', { const msg = JSON.parse(message.toString()) const sessionId = this.sessionId - if (parsedTopic.tool === 'expert:status-message') { + if (parsedTopic.inflightType === 'expert:status-message') { const responseTopic = topicHelper.buildTopic({ entityType: parsedTopic.entityType, entityId: parsedTopic.entityId, agentChannel: 'support', topicType: 'inflight', - topicAction: 'response' + topicAction: 'response', + inflightType: parsedTopic.inflightType }) // this is just a status from the agent, we can ignore it for now, but we might want to display it in the UI later @@ -341,10 +355,10 @@ export const useProductExpertStore = defineStore('product-expert', { correlationData: new TextEncoder().encode(transactionId), userProperties: { sessionId } }) - } else if (parsedTopic.tool.startsWith('automation:')) { + } else if (parsedTopic.inflightType.startsWith('automation:')) { // thi is an automation request, we want to execute it and return the result to the agent const postMessagePayload = { - action: parsedTopic.tool.replace('automation:', 'automation/'), + action: parsedTopic.inflightType.replace('automation:', 'automation/'), params: msg.payload?.params || {}, userProperties: { sessionId }, transactionId @@ -363,19 +377,23 @@ export const useProductExpertStore = defineStore('product-expert', { const mqttService = servicesOrchestrator.$serviceInstances.mqtt const topicHelper = useMqttExpertTopicHelper() - const originalTopic = topicHelper.parseTopic() + const originalInFlightRequest = this._inFlightRequests.get(transactionId) + const originalTopic = topicHelper.parseTopic(originalInFlightRequest.topic) const replyTopic = topicHelper.buildTopic({ entityType: originalTopic.entityType, entityId: originalTopic.entityId, agentChannel: 'support', topicType: 'inflight', - topicAction: 'response' + topicAction: 'response', + inflightType: originalTopic.inflightType }) + const payload = JSON.stringify(response.data) + await mqttService.publishMessage(this.mqttConnectionKey, { qos: 2, topic: replyTopic, - payload: JSON.stringify(response), + payload, correlationData: new TextEncoder().encode(transactionId), userProperties: { sessionId: this.sessionId } }) From f1f63c81d8875f7756d950bd76e6cbba53554395 Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Mon, 20 Apr 2026 13:09:50 +0100 Subject: [PATCH 11/54] Add pendingRequests to product assistant store for managing asynchronous operations --- frontend/src/stores/product-assistant.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/stores/product-assistant.js b/frontend/src/stores/product-assistant.js index 913bf4438b..810eedfc1a 100644 --- a/frontend/src/stores/product-assistant.js +++ b/frontend/src/stores/product-assistant.js @@ -158,7 +158,8 @@ export const useProductAssistantStore = defineStore('product-assistant', { // debugLog getter which gates the feature flag. Use this.debugLogEntries // internally; external consumers should read this.debugLog (the getter). debugLogEntries: [], - editorState: { ...buildInitialEditorState() } + editorState: { ...buildInitialEditorState() }, + pendingRequests: new Map() // key is transactionId, value is { resolve, reject, timeout, timestamp, type, action, params } }), getters: { immersiveInstance: () => { From d71d8924a8dc2f2bc91cf8bd7323404e3c21c45e Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Mon, 20 Apr 2026 13:11:06 +0100 Subject: [PATCH 12/54] Handle responses for in-flight requests and add invokeActionAwaitResponse method for awaiting responses --- frontend/src/stores/product-assistant.js | 70 ++++++++++++++++++++---- 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/frontend/src/stores/product-assistant.js b/frontend/src/stores/product-assistant.js index 810eedfc1a..f24a2f12a6 100644 --- a/frontend/src/stores/product-assistant.js +++ b/frontend/src/stores/product-assistant.js @@ -3,7 +3,6 @@ import SemVer from 'semver' import getServicesOrchestrator from '@/services/service.orchestrator' import { useContextStore } from '@/stores/context.js' -import { useProductExpertStore } from '@/stores/product-expert.js' const MAX_DEBUG_LOG_ENTRIES = 100 // maximum number of debug log entries to keep @@ -270,13 +269,17 @@ export const useProductAssistantStore = defineStore('product-assistant', { return } - // todo define how we receive the transaction id back - if (payload.data.transactionId) { - const expertStore = useProductExpertStore() - await expertStore.handleAgentReply({ - transactionId: payload.data.transactionId, - response: payload - }) + const payloadData = payload.data || {} + if (payloadData.correlationId && payloadData.type === 'invoke-action') { + const { correlationId } = payloadData + const inflight = this.pendingRequests.get(correlationId) + if (inflight) { + console.debug('Received response for in-flight request:', { correlationId, originalRequest: inflight.postMessagePayload, response: payloadData }) + inflight.resolve(payloadData) + this.pendingRequests.delete(correlationId) + } else { + console.warn('Received response with correlationId that does not match any in-flight requests. Ignoring.', { correlationId, response: payloadData }) + } } switch (true) { @@ -459,14 +462,59 @@ export const useProductAssistantStore = defineStore('product-assistant', { resetContextSelection () { this.selectedContext = this.availableContextOptions }, - async invokeAction ({ action, params, userProperties, transactionId }) { + async invokeAction ({ action, params }) { return this.sendMessage({ + type: 'invoke-action', + action, + params + }) + }, + async invokeActionAwaitResponse ({ action, params, sessionId, transactionId, chatTransactionId }, timeout = 1000) { + // create a promise that will resolve when we receive a response with the matching sessionId and transactionId, or reject after a timeout + const pending = { + resolve: null, + reject: null, + resolved: false, + rejected: false, + timeout: null, + timestamp: Date.now(), + type: 'invoke-action', + action, + params, + sessionId, + transactionId, + chatTransactionId + } + const promise = new Promise((resolve, reject) => { + pending.resolve = (payload) => { + clearTimeout(pending.timeout) + this.pendingRequests.delete(pending.transactionId) + if (pending.resolved) return + if (pending.rejected) return + pending.resolved = true + resolve(payload) + } + pending.reject = (error) => { + clearTimeout(pending.timeout) + this.pendingRequests.delete(pending.transactionId) + if (pending.resolved) return + if (pending.rejected) return + pending.rejected = true + reject(error) + } + pending.timeout = setTimeout(() => { + pending.reject(new Error('Command timed out')) + }, timeout) + }) + const correlationId = `${sessionId}:${chatTransactionId}:${transactionId}` + this.pendingRequests.set(correlationId, pending) + this.sendMessage({ type: 'invoke-action', action, params, - userProperties, - transactionId + correlationId // post Message Correlation Id }) + return promise }, async sendFlowsToImport (flowsJson) { return this.sendMessage({ From e58ac9fcdc73faf599f94e73b399491a2381470e Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Mon, 20 Apr 2026 13:16:58 +0100 Subject: [PATCH 13/54] Use invoke await to ensure assistant transaction is correlated --- frontend/src/stores/product-expert.js | 86 ++++++++++----------------- 1 file changed, 33 insertions(+), 53 deletions(-) diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 4ce17a48e1..8cdfac3340 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -215,6 +215,8 @@ export const useProductExpertStore = defineStore('product-expert', { onMessage: async (topic, message, packet) => { const parsedTopic = topicHelper.parseTopic(topic) const transactionId = packet.properties?.correlationData ? new TextDecoder().decode(packet.properties.correlationData) : null + const sessionId = packet.properties?.userProperties?.sessionId // chat session + const chatTransactionId = packet.properties?.userProperties?.transactionId // passed through for integrity if (parsedTopic.isReply) { // remove inFlight request because it is now resolved @@ -227,7 +229,9 @@ export const useProductExpertStore = defineStore('product-expert', { topic, message, packet, - transactionId + transactionId, + sessionId, + chatTransactionId }) } }, @@ -321,7 +325,7 @@ export const useProductExpertStore = defineStore('product-expert', { } }) }, - async handleInFlightRequest ({ topic, message, packet, transactionId }) { + async handleInFlightRequest ({ topic, message, packet, transactionId, sessionId, chatTransactionId }) { const servicesOrchestrator = getServicesOrchestrator() const mqttService = servicesOrchestrator.$serviceInstances.mqtt @@ -330,22 +334,22 @@ export const useProductExpertStore = defineStore('product-expert', { const topicHelper = useMqttExpertTopicHelper() const parsedTopic = topicHelper.parseTopic(topic) - const msg = JSON.parse(message.toString()) - const sessionId = this.sessionId + const payload = JSON.parse(message.toString()) - if (parsedTopic.inflightType === 'expert:status-message') { - const responseTopic = topicHelper.buildTopic({ - entityType: parsedTopic.entityType, - entityId: parsedTopic.entityId, - agentChannel: 'support', - topicType: 'inflight', - topicAction: 'response', - inflightType: parsedTopic.inflightType - }) + const responseTopic = topicHelper.buildTopic({ + entityType: parsedTopic.entityType, + entityId: parsedTopic.entityId, + agentChannel: 'support', + topicType: 'inflight', + topicAction: 'response', + inflightType: parsedTopic.inflightType, + sessionId: sessionId || this.sessionId // TODO: consider validating the incoming request session in userProps matches + }) + if (parsedTopic.inflightType === 'expert:status-message') { // this is just a status from the agent, we can ignore it for now, but we might want to display it in the UI later // just ack the msg - // todo do we need it? + // TODO: We should be updating the message in the chat interface with _busy doing xyz_ status updates instead of the made up text like "Reading Rag", "Calling MCP" etc. await mqttService.publishMessage(this.mqttConnectionKey, { qos: 2, topic: responseTopic, @@ -353,53 +357,29 @@ export const useProductExpertStore = defineStore('product-expert', { ack: true }), correlationData: new TextEncoder().encode(transactionId), - userProperties: { sessionId } + userProperties: { sessionId, transactionId: chatTransactionId } // pass through for integrity, not actually needed for correlation in this case since it's just an ack }) } else if (parsedTopic.inflightType.startsWith('automation:')) { // thi is an automation request, we want to execute it and return the result to the agent + const actionName = parsedTopic.inflightType.replace('automation:', '') const postMessagePayload = { - action: parsedTopic.inflightType.replace('automation:', 'automation/'), - params: msg.payload?.params || {}, - userProperties: { sessionId }, - transactionId + action: `automation/${actionName}`, + params: payload.params || {}, + sessionId, // the chat session + chatTransactionId, // the chat transaction, passed through for integrity + transactionId // the incoming MQTT transaction (not the original chat transaction - we store that in userProperties) } - // adding the inFlightRequest to the queue because the message is passed along to the nr-assistant - this._inFlightRequests.set(transactionId, { topic, message, packet, postMessagePayload }) - - await assistantStore.invokeAction(postMessagePayload) + const result = await assistantStore.invokeActionAwaitResponse(postMessagePayload) + await mqttService.publishMessage(this.mqttConnectionKey, { + qos: 2, + topic: responseTopic, + payload: JSON.stringify(result), + correlationData: new TextEncoder().encode(transactionId), + userProperties: { sessionId, transactionId: chatTransactionId } // pass through for integrity + }) } }, - async handleAgentReply ({ transactionId, response }) { - if (!this._inFlightRequests.has(transactionId)) return false - - const servicesOrchestrator = getServicesOrchestrator() - const mqttService = servicesOrchestrator.$serviceInstances.mqtt - const topicHelper = useMqttExpertTopicHelper() - - const originalInFlightRequest = this._inFlightRequests.get(transactionId) - const originalTopic = topicHelper.parseTopic(originalInFlightRequest.topic) - const replyTopic = topicHelper.buildTopic({ - entityType: originalTopic.entityType, - entityId: originalTopic.entityId, - agentChannel: 'support', - topicType: 'inflight', - topicAction: 'response', - inflightType: originalTopic.inflightType - }) - - const payload = JSON.stringify(response.data) - - await mqttService.publishMessage(this.mqttConnectionKey, { - qos: 2, - topic: replyTopic, - payload, - correlationData: new TextEncoder().encode(transactionId), - userProperties: { sessionId: this.sessionId } - }) - - this._inFlightRequests.delete(transactionId) - }, async handleMessageResponse (response) { if (response.answer && Array.isArray(response.answer)) { this.addAiMessage(response) From 54bd46e55a3770598f0f5ce5653187e99c0d31fe Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 20 Apr 2026 13:55:59 +0100 Subject: [PATCH 14/54] Enable TeamBroker for pre-staging --- ci/ci-values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/ci-values.yaml b/ci/ci-values.yaml index 05eda6c164..925632da7e 100644 --- a/ci/ci-values.yaml +++ b/ci/ci-values.yaml @@ -7,7 +7,7 @@ forge: broker: enabled: true teamBroker: - uiOnly: true + enabled: true cloudProvider: aws cache: type: memory From 3c2c0edcc091e4ec755e678aa4629792f3aac833 Mon Sep 17 00:00:00 2001 From: cstns Date: Mon, 20 Apr 2026 18:38:13 +0300 Subject: [PATCH 15/54] increase sendMessage await timeout --- frontend/src/stores/product-assistant.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/stores/product-assistant.js b/frontend/src/stores/product-assistant.js index f24a2f12a6..e7802720ed 100644 --- a/frontend/src/stores/product-assistant.js +++ b/frontend/src/stores/product-assistant.js @@ -469,7 +469,7 @@ export const useProductAssistantStore = defineStore('product-assistant', { params }) }, - async invokeActionAwaitResponse ({ action, params, sessionId, transactionId, chatTransactionId }, timeout = 1000) { + async invokeActionAwaitResponse ({ action, params, sessionId, transactionId, chatTransactionId }, timeout = 5000) { // create a promise that will resolve when we receive a response with the matching sessionId and transactionId, or reject after a timeout const pending = { resolve: null, From 636b76f917202ed99aea9b7a818a207cd10bbdbd Mon Sep 17 00:00:00 2001 From: cstns Date: Mon, 20 Apr 2026 19:31:18 +0300 Subject: [PATCH 16/54] disjoint onEvent callbacks to avoid recursion --- frontend/src/stores/product-expert.js | 235 +++++++++++++------------- 1 file changed, 119 insertions(+), 116 deletions(-) diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 8cdfac3340..16521baba1 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -205,124 +205,14 @@ export const useProductExpertStore = defineStore('product-expert', { async establishMqttComms () { const servicesOrchestrator = getServicesOrchestrator() const mqttService = servicesOrchestrator.$serviceInstances.mqtt - const topicHelper = useMqttExpertTopicHelper() - - // todo remove, dev only - window.qwe = mqttService await mqttService.createClient(this.mqttConnectionKey, { getCredentials: () => userApi.initiateExpertChat({ sessionId: this.sessionId }), - onMessage: async (topic, message, packet) => { - const parsedTopic = topicHelper.parseTopic(topic) - const transactionId = packet.properties?.correlationData ? new TextDecoder().decode(packet.properties.correlationData) : null - const sessionId = packet.properties?.userProperties?.sessionId // chat session - const chatTransactionId = packet.properties?.userProperties?.transactionId // passed through for integrity - - if (parsedTopic.isReply) { - // remove inFlight request because it is now resolved - this._inFlightRequests.delete(transactionId) - - // handle the response - await this.handleMessageResponse(JSON.parse(message.toString())) - } else if (parsedTopic.isInflightRequest) { - await this.handleInFlightRequest({ - topic, - message, - packet, - transactionId, - sessionId, - chatTransactionId - }) - } - }, - onClose: () => { - // TODO add error message - }, - onConnect: () => { - const mqttTopicHelper = useMqttExpertTopicHelper() - - mqttService.subscribe( - this.mqttConnectionKey, - mqttTopicHelper.buildTopic({ - entityType: '+', - entityId: '+', - agentChannel: 'support', - topicType: 'chat', - topicAction: 'response' - }), - { qos: 2 } - ) - - mqttService.subscribe( - this.mqttConnectionKey, - mqttTopicHelper.buildTopic({ - entityType: '+', - entityId: '+', - agentChannel: 'support', - topicType: 'inflight', - topicAction: 'request', - inflightType: '+' - }), - { qos: 2 } - ) - - // whenever the sessionId changes, we need to unsubscribe from previous topics and subscribe to the - // new ones based off of the new sessionId - watch( - () => this._agentStore.sessionId, - async () => { - if (!mqttService.hasClient(this.mqttConnectionKey)) return - - const timerHelper = useTimerHelper() - await mqttService.destroyClient(this.mqttConnectionKey) - - // todo extract all hooks into atomic methods and prevent this recursion from happening - // good enough for demo purposes - await this.establishMqttComms() - // todo also, getting required creds fails from time to time because we're creating the client - // before the backend successfully remove the old one - - const managedClient = mqttService.getManagedClient(this.mqttConnectionKey) - - await timerHelper.waitWhile( - () => ['connected'].includes(managedClient.status), - { intervalMs: 200, cutoffTries: 50 } - ) - - await new Promise(resolve => setTimeout(resolve, 5000)) - - await mqttService.subscribe( - this.mqttConnectionKey, - mqttTopicHelper.buildTopic({ - entityType: '+', - entityId: '+', - agentChannel: 'support', - topicType: 'chat', - topicAction: 'response' - }), - { qos: 2 } - ) - await mqttService.subscribe( - this.mqttConnectionKey, - mqttTopicHelper.buildTopic({ - entityType: '+', - entityId: '+', - agentChannel: 'support', - topicType: 'inflight', - topicAction: 'request', - inflightType: '+' - }), - { qos: 2 } - ) - } - ) - }, - onOffline: () => { - // TODO add error message - }, - onError: (e) => { - // TODO add error message - } + onMessage: this._onMqttMessage, + onClose: this._onMqttClose, + onConnect: this._onMqttConnect, + onOffline: this._onMqttOffline, + onError: this._onMqttError }) }, async handleInFlightRequest ({ topic, message, packet, transactionId, sessionId, chatTransactionId }) { @@ -601,7 +491,7 @@ export const useProductExpertStore = defineStore('product-expert', { hydrateMessages (messages) { if (!messages) return messages.forEach((message) => { - // todo break this into manageable chunks, do we actually still need it? + // TODO break this into manageable chunks, do we actually still need it? if (message.answer && Array.isArray(message.answer)) { // Extract MCP items (tools, resources, resource templates, prompts) from the answer array const mcpItems = message.answer.filter(item => @@ -651,6 +541,119 @@ export const useProductExpertStore = defineStore('product-expert', { } // Else: ignore messages that don't match either format }) + }, + async _onMqttMessage (topic, message, packet) { + const topicHelper = useMqttExpertTopicHelper() + const parsedTopic = topicHelper.parseTopic(topic) + const transactionId = packet.properties?.correlationData ? new TextDecoder().decode(packet.properties.correlationData) : null + const sessionId = packet.properties?.userProperties?.sessionId // chat session + const chatTransactionId = packet.properties?.userProperties?.transactionId // passed through for integrity + + if (parsedTopic.isReply) { + // remove inFlight request because it is now resolved + this._inFlightRequests.delete(transactionId) + + // handle the response + await this.handleMessageResponse(JSON.parse(message.toString())) + } else if (parsedTopic.isInflightRequest) { + await this.handleInFlightRequest({ + topic, + message, + packet, + transactionId, + sessionId, + chatTransactionId + }) + } + }, + _onMqttClose () { + // TODO add error message + }, + _onMqttConnect () { + const servicesOrchestrator = getServicesOrchestrator() + const mqttService = servicesOrchestrator.$serviceInstances.mqtt + + const mqttTopicHelper = useMqttExpertTopicHelper() + + mqttService.subscribe( + this.mqttConnectionKey, + mqttTopicHelper.buildTopic({ + entityType: '+', + entityId: '+', + agentChannel: 'support', + topicType: 'chat', + topicAction: 'response' + }), + { qos: 2 } + ) + + mqttService.subscribe( + this.mqttConnectionKey, + mqttTopicHelper.buildTopic({ + entityType: '+', + entityId: '+', + agentChannel: 'support', + topicType: 'inflight', + topicAction: 'request', + inflightType: '+' + }), + { qos: 2 } + ) + + // whenever the sessionId changes, we need to unsubscribe from previous topics and subscribe to the + // new ones based off of the new sessionId + watch( + () => this._agentStore.sessionId, + async () => { + if (!mqttService.hasClient(this.mqttConnectionKey)) return + + const timerHelper = useTimerHelper() + await mqttService.destroyClient(this.mqttConnectionKey) + + // TODO getting required creds fails from time to time because we're creating the client + // before the backend successfully remove the old one + await this.establishMqttComms() + + const managedClient = mqttService.getManagedClient(this.mqttConnectionKey) + + await timerHelper.waitWhile( + () => ['connected'].includes(managedClient.status), + { intervalMs: 200, cutoffTries: 50 } + ) + + await new Promise(resolve => setTimeout(resolve, 5000)) + + await mqttService.subscribe( + this.mqttConnectionKey, + mqttTopicHelper.buildTopic({ + entityType: '+', + entityId: '+', + agentChannel: 'support', + topicType: 'chat', + topicAction: 'response' + }), + { qos: 2 } + ) + await mqttService.subscribe( + this.mqttConnectionKey, + mqttTopicHelper.buildTopic({ + entityType: '+', + entityId: '+', + agentChannel: 'support', + topicType: 'inflight', + topicAction: 'request', + inflightType: '+' + }), + { qos: 2 } + ) + } + ) + }, + _onMqttOffline () { + // TODO add error message + }, + _onMqttError (e) { + // TODO add error message } }, persist: { From 169a3e7a96f0a4ffd27b5b16de612dfd9a1519fd Mon Sep 17 00:00:00 2001 From: cstns Date: Mon, 20 Apr 2026 20:47:00 +0300 Subject: [PATCH 17/54] update the loading indicator to take mqtt inflight requests, update the loading messages payloads with tool names and show a default error message when something goes wrong --- .../components/ExpertLoadingIndicator.vue | 6 ++- frontend/src/stores/product-assistant.js | 2 +- frontend/src/stores/product-expert.js | 46 ++++++++++++++----- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/frontend/src/components/expert/components/ExpertLoadingIndicator.vue b/frontend/src/components/expert/components/ExpertLoadingIndicator.vue index ceb9c02062..22921e9480 100644 --- a/frontend/src/components/expert/components/ExpertLoadingIndicator.vue +++ b/frontend/src/components/expert/components/ExpertLoadingIndicator.vue @@ -53,8 +53,12 @@ export default { } }, computed: { - ...mapState(useProductExpertStore, ['loadingVariant']), + ...mapState(useProductExpertStore, ['loadingVariant', 'inFlightUpdates']), messages () { + if (this.inFlightUpdates.length > 0) { + return this.inFlightUpdates + } + return this.messageVariants[this.loadingVariant] }, currentMessage () { diff --git a/frontend/src/stores/product-assistant.js b/frontend/src/stores/product-assistant.js index e7802720ed..779ec73a27 100644 --- a/frontend/src/stores/product-assistant.js +++ b/frontend/src/stores/product-assistant.js @@ -503,7 +503,7 @@ export const useProductAssistantStore = defineStore('product-assistant', { reject(error) } pending.timeout = setTimeout(() => { - pending.reject(new Error('Command timed out')) + pending.reject(new Error('Node-RED command timed out')) }, timeout) }) const correlationId = `${sessionId}:${chatTransactionId}:${transactionId}` diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 16521baba1..532ce28c29 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -1,6 +1,6 @@ import { defineStore } from 'pinia' import { v4 as uuidv4 } from 'uuid' -import { markRaw, watch } from 'vue' +import { UnwrapRef, markRaw, watch } from 'vue' import expertApi from '../api/expert.js' import userApi from '../api/user.js' @@ -25,7 +25,8 @@ export const useProductExpertStore = defineStore('product-expert', { state: () => ({ agentMode: SUPPORT_AGENT, // support-agent or insights-agent loadingVariant: SUPPORT_AGENT, - shouldWakeUpAssistant: false + shouldWakeUpAssistant: false, + inFlightUpdates: [] }), getters: { _agentStore () { @@ -33,6 +34,10 @@ export const useProductExpertStore = defineStore('product-expert', { ? useProductExpertSupportAgentStore() : useProductExpertInsightsAgentStore() }, + /** + * @return {UnwrapRef | Map>|UnwrapRef | Map>} + * @private + */ _inFlightRequests () { return this.agentMode === SUPPORT_AGENT ? useProductExpertSupportAgentStore().inFlightRequests @@ -42,7 +47,7 @@ export const useProductExpertStore = defineStore('product-expert', { messages () { return this._agentStore.messages }, hasMessages () { return this._agentStore.messages.length > 0 }, isSessionExpired () { return this._agentStore.sessionExpiredShown }, - isWaitingForResponse () { return !!this._agentStore.abortController }, + isWaitingForResponse () { return !!this._agentStore.abortController || this._inFlightRequests.size > 0 }, isSupportAgent: (state) => state.agentMode === SUPPORT_AGENT, isInsightsAgent: (state) => state.agentMode === INSIGHTS_AGENT, hasSelectedCapabilities () { @@ -226,6 +231,8 @@ export const useProductExpertStore = defineStore('product-expert', { const payload = JSON.parse(message.toString()) + this._addInFlightUpdate(payload.toolname) + const responseTopic = topicHelper.buildTopic({ entityType: parsedTopic.entityType, entityId: parsedTopic.entityId, @@ -260,14 +267,18 @@ export const useProductExpertStore = defineStore('product-expert', { transactionId // the incoming MQTT transaction (not the original chat transaction - we store that in userProperties) } - const result = await assistantStore.invokeActionAwaitResponse(postMessagePayload) - await mqttService.publishMessage(this.mqttConnectionKey, { - qos: 2, - topic: responseTopic, - payload: JSON.stringify(result), - correlationData: new TextEncoder().encode(transactionId), - userProperties: { sessionId, transactionId: chatTransactionId } // pass through for integrity - }) + try { + const result = await assistantStore.invokeActionAwaitResponse(postMessagePayload) + await mqttService.publishMessage(this.mqttConnectionKey, { + qos: 2, + topic: responseTopic, + payload: JSON.stringify(result), + correlationData: new TextEncoder().encode(transactionId), + userProperties: { sessionId, transactionId: chatTransactionId } // pass through for integrity + }) + } catch (e) { + this._onMqttError(e) + } } }, async handleMessageResponse (response) { @@ -653,7 +664,18 @@ export const useProductExpertStore = defineStore('product-expert', { // TODO add error message }, _onMqttError (e) { - // TODO add error message + this._inFlightRequests.clear() + this._clearInFlightUpdates() + this.addPredefinedAiMessage(`Something went wrong.. ${e.message}`) + // TODO, by this point we might even ignore any other inflight requests which might be in the pipeline, but + // that has to be done from somewhere else.. maybe we should ignore all inflight requests if their original + // correlationId doesn't match the initial correlationId sent out on the query + }, + _addInFlightUpdate (status) { + this.inFlightUpdates.push(status) + }, + _clearInFlightUpdates () { + this.inFlightUpdates = [] } }, persist: { From f99c1b950ba978270b1e53a779d90d406afefb30 Mon Sep 17 00:00:00 2001 From: cstns Date: Tue, 21 Apr 2026 11:35:01 +0300 Subject: [PATCH 18/54] stop inflight mqtt requests --- frontend/src/components/expert/Expert.vue | 4 +- frontend/src/stores/product-expert.js | 91 +++++++++++++++++++---- 2 files changed, 81 insertions(+), 14 deletions(-) diff --git a/frontend/src/components/expert/Expert.vue b/frontend/src/components/expert/Expert.vue index f40c17ab36..406248a9ea 100644 --- a/frontend/src/components/expert/Expert.vue +++ b/frontend/src/components/expert/Expert.vue @@ -148,7 +148,8 @@ export default { 'setAgentMode', 'setAbortController', 'resetSessionTimer', - 'addWelcomeMessageIfNeeded' + 'addWelcomeMessageIfNeeded', + 'stopInflightChat' ]), ...mapActions(useProductExpertInsightsAgentStore, ['getCapabilities']), ...mapActions(useProductAssistantStore, ['reset']), @@ -157,6 +158,7 @@ export default { this.abortController.abort() this.setAbortController(null) } + this.stopInflightChat() }, handleScroll () { // Debounce scroll detection diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 532ce28c29..1a8f61ed7c 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -62,7 +62,8 @@ export const useProductExpertStore = defineStore('product-expert', { return !!assistantStore.immersiveInstance && !!assistantStore.supportedActions['core:manage-palette'] }, mqttConnectionKey () { return this._agentStore.mqttConnectionKey }, - sessionId () { return this._agentStore.sessionId } + sessionId () { return this._agentStore.sessionId }, + shouldUseMqtt () { return IS_MQTT_ENABLED && this.isSupportAgent } }, actions: { setContext ({ data, sessionId }) { @@ -145,8 +146,8 @@ export const useProductExpertStore = defineStore('product-expert', { } }, sendQuery ({ query }) { - if (IS_MQTT_ENABLED && this.isSupportAgent) { - return this.handleMqttQuery({ query }) + if (this.shouldUseMqtt) { + return this.sendMqttQuery({ query }) } else { return this.sendHttpQuery({ query }) } @@ -169,7 +170,7 @@ export const useProductExpertStore = defineStore('product-expert', { return expertApi.chat(payload) }, - async handleMqttQuery ({ query }) { + async sendMqttQuery ({ query } = {}) { const servicesOrchestrator = getServicesOrchestrator() const mqttService = servicesOrchestrator.$serviceInstances.mqtt const mqttTopicHelper = useMqttExpertTopicHelper() @@ -177,10 +178,11 @@ export const useProductExpertStore = defineStore('product-expert', { const transactionId = uuidv4() const mqttConnectionKey = this.mqttConnectionKey + if (!mqttService.hasClient(mqttConnectionKey)) await this.establishMqttComms() + // add the query as an inFlight request - this._inFlightRequests.set(transactionId, { query }) + this._inFlightRequests.set(transactionId, { query, transactionId }) - if (!mqttService.hasClient(mqttConnectionKey)) await this.establishMqttComms() const { entityId, entityType } = mqttTopicHelper.getEntityTopicPaths() const topic = mqttTopicHelper.buildTopic({ @@ -220,7 +222,19 @@ export const useProductExpertStore = defineStore('product-expert', { onError: this._onMqttError }) }, - async handleInFlightRequest ({ topic, message, packet, transactionId, sessionId, chatTransactionId }) { + async handleInFlightRequest ({ topic, message, packet, transactionId, sessionId, chatTransactionId } = {}) { + // console.log('handleInFlightRequest', { + // topic, + // message, + // packet, + // transactionId, + // sessionId, + // chatTransactionId + // }) + + const inFlightRequest = this._inFlightRequests.values().next().value + if (sessionId !== this.sessionId || inFlightRequest?.transactionId !== chatTransactionId) return + const servicesOrchestrator = getServicesOrchestrator() const mqttService = servicesOrchestrator.$serviceInstances.mqtt @@ -282,6 +296,11 @@ export const useProductExpertStore = defineStore('product-expert', { } }, async handleMessageResponse (response) { + // console.log('handling message response before validation: ', { response }) + + // ignore aborted messages through mqtt + if (Object.prototype.hasOwnProperty.call(response, 'aborted') && response.aborted === true) return + if (response.answer && Array.isArray(response.answer)) { this.addAiMessage(response) } @@ -449,8 +468,13 @@ export const useProductExpertStore = defineStore('product-expert', { }) }, addPredefinedAiMessage (message) { + // this may not be the best approach + // if the last entry in messages is also auto generated, skip, most probably we'll be adding a duplication + if (this._agentStore.messages.length && this._agentStore.messages.at(-1).generated) return + this._agentStore.messages.push({ _type: 'ai', + generated: true, answer: [{ content: message, _streamed: false, _uuid: uuidv4() }], _timestamp: Date.now(), _streamed: false, @@ -554,6 +578,9 @@ export const useProductExpertStore = defineStore('product-expert', { }) }, async _onMqttMessage (topic, message, packet) { + // ignore any messages if inFlightRequests has been cleared (it means that the chat was stopped mid-flight) + if (this._inFlightRequests.size === 0) return + const topicHelper = useMqttExpertTopicHelper() const parsedTopic = topicHelper.parseTopic(topic) const transactionId = packet.properties?.correlationData ? new TextDecoder().decode(packet.properties.correlationData) : null @@ -578,7 +605,10 @@ export const useProductExpertStore = defineStore('product-expert', { } }, _onMqttClose () { - // TODO add error message + this.addPredefinedAiMessage( + 'Something went wrong and our connection was interrupted. ' + + 'You can continue this conversation by sending a new message, or simply start over with a new session.' + ) }, _onMqttConnect () { const servicesOrchestrator = getServicesOrchestrator() @@ -661,21 +691,56 @@ export const useProductExpertStore = defineStore('product-expert', { ) }, _onMqttOffline () { - // TODO add error message + console.log('#################### mqtt offline') + // TODO add error message, handle reconnect, notify user }, _onMqttError (e) { - this._inFlightRequests.clear() + // stopping inFlight chat requests to ignore any in flight messages + this.stopInflightChat() this._clearInFlightUpdates() this.addPredefinedAiMessage(`Something went wrong.. ${e.message}`) - // TODO, by this point we might even ignore any other inflight requests which might be in the pipeline, but - // that has to be done from somewhere else.. maybe we should ignore all inflight requests if their original - // correlationId doesn't match the initial correlationId sent out on the query }, _addInFlightUpdate (status) { this.inFlightUpdates.push(status) }, _clearInFlightUpdates () { this.inFlightUpdates = [] + }, + stopInflightChat () { + if (this.shouldUseMqtt) { + const servicesOrchestrator = getServicesOrchestrator() + const mqttService = servicesOrchestrator.$serviceInstances.mqtt + const mqttTopicHelper = useMqttExpertTopicHelper() + const inFlightRequest = this._inFlightRequests.values().next().value + + const { entityId, entityType } = mqttTopicHelper.getEntityTopicPaths() + + const topic = mqttTopicHelper.buildTopic({ + entityType, + entityId, + agentChannel: 'support', + topicType: 'chat', + topicAction: 'request' + }) + + // publishing an abort message to stop the agent + return mqttService.publishMessage(this.mqttConnectionKey, { + topic, + qos: 2, + payload: { + abort: true, + context: { + ...useContextStore().expert, + agent: this.agentMode + } + }, + correlationData: inFlightRequest.transactionId, + userProperties: { + sessionId: this.sessionId + } + }) + } + this._inFlightRequests.clear() } }, persist: { From 8b45ba6ffdf955f4f29e2fa74d869c8efde25707 Mon Sep 17 00:00:00 2001 From: cstns Date: Tue, 21 Apr 2026 12:21:05 +0300 Subject: [PATCH 19/54] stop inflight mqtt requests --- frontend/src/stores/product-expert.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 1a8f61ed7c..aade82d580 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -724,7 +724,7 @@ export const useProductExpertStore = defineStore('product-expert', { }) // publishing an abort message to stop the agent - return mqttService.publishMessage(this.mqttConnectionKey, { + mqttService.publishMessage(this.mqttConnectionKey, { topic, qos: 2, payload: { From 3eba77703de166fc4a95f7873fb207b303a536dd Mon Sep 17 00:00:00 2001 From: cstns Date: Tue, 21 Apr 2026 12:48:38 +0300 Subject: [PATCH 20/54] add expert:status-messages into the inFlightUpdates payload for more accurate feedback --- frontend/src/stores/product-expert.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index aade82d580..459fb8a596 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -245,7 +245,7 @@ export const useProductExpertStore = defineStore('product-expert', { const payload = JSON.parse(message.toString()) - this._addInFlightUpdate(payload.toolname) + this._addInFlightUpdate(payload.toolname || payload.status) const responseTopic = topicHelper.buildTopic({ entityType: parsedTopic.entityType, From a2c40adf712d0d4d3c4225531b2b203bd3ec01e6 Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Tue, 21 Apr 2026 11:48:09 +0100 Subject: [PATCH 21/54] prioritize status over tool name --- frontend/src/stores/product-expert.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 459fb8a596..5815d9ac06 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -245,7 +245,7 @@ export const useProductExpertStore = defineStore('product-expert', { const payload = JSON.parse(message.toString()) - this._addInFlightUpdate(payload.toolname || payload.status) + this._addInFlightUpdate(payload.status || payload.toolname || 'Processing request...') const responseTopic = topicHelper.buildTopic({ entityType: parsedTopic.entityType, From 7cc8e7175ff5bd5d433ea6a2c018a6fbaf99cc1f Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Tue, 21 Apr 2026 11:48:38 +0100 Subject: [PATCH 22/54] Fix lint for unused type --- frontend/src/stores/product-expert.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 5815d9ac06..226d393098 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -1,6 +1,6 @@ import { defineStore } from 'pinia' import { v4 as uuidv4 } from 'uuid' -import { UnwrapRef, markRaw, watch } from 'vue' +import { markRaw, watch } from 'vue' import expertApi from '../api/expert.js' import userApi from '../api/user.js' @@ -35,7 +35,7 @@ export const useProductExpertStore = defineStore('product-expert', { : useProductExpertInsightsAgentStore() }, /** - * @return {UnwrapRef | Map>|UnwrapRef | Map>} + * @return {import('vue').UnwrapRef | Map> | import('vue').UnwrapRef | Map>} * @private */ _inFlightRequests () { From 051bf0aa256a44e85c73ff936f97f10183fc9854 Mon Sep 17 00:00:00 2001 From: cstns Date: Tue, 21 Apr 2026 14:25:31 +0300 Subject: [PATCH 23/54] clear inflight updates after mqtt response --- frontend/src/stores/product-expert.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 226d393098..e7b77aa738 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -296,13 +296,12 @@ export const useProductExpertStore = defineStore('product-expert', { } }, async handleMessageResponse (response) { - // console.log('handling message response before validation: ', { response }) - // ignore aborted messages through mqtt if (Object.prototype.hasOwnProperty.call(response, 'aborted') && response.aborted === true) return if (response.answer && Array.isArray(response.answer)) { this.addAiMessage(response) + this._clearInFlightUpdates() } }, async startOver () { From e551de680dec8a83654c633c5ca7d8ac22ee58d4 Mon Sep 17 00:00:00 2001 From: cstns Date: Tue, 21 Apr 2026 14:52:35 +0300 Subject: [PATCH 24/54] improve table styling for md tables in ai responses --- .../resources/StreamableContent.vue | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/frontend/src/components/expert/components/messages/components/resources/StreamableContent.vue b/frontend/src/components/expert/components/messages/components/resources/StreamableContent.vue index 436250a8cc..b4d31b638f 100644 --- a/frontend/src/components/expert/components/messages/components/resources/StreamableContent.vue +++ b/frontend/src/components/expert/components/messages/components/resources/StreamableContent.vue @@ -222,5 +222,27 @@ export default { background-color: $ff-color--highlight; } } + + :deep(table) { + border-collapse: collapse; /* removes double borders */ + + tr + tr td { + font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace; + border-top: 1px solid $ff-grey-200; + } + + td, th { + padding: 5px 10px; + + } + + td { + code { + padding: 0; + border: none; + border-radius: 0; + } + } + } } From 728823121cfb708bbf54188752651c43e460cccb Mon Sep 17 00:00:00 2001 From: cstns Date: Tue, 21 Apr 2026 15:38:17 +0300 Subject: [PATCH 25/54] makes the tool list scrollable and expandable, a max length of 5 when collapsed and a max height of 15 tools when expanded --- .../expert/components/messages/AiMessage.vue | 7 ++++--- .../components/messages/components/ToolCalls.vue | 10 +++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/expert/components/messages/AiMessage.vue b/frontend/src/components/expert/components/messages/AiMessage.vue index b38503b269..0fc8817324 100644 --- a/frontend/src/components/expert/components/messages/AiMessage.vue +++ b/frontend/src/components/expert/components/messages/AiMessage.vue @@ -12,12 +12,13 @@ + + diff --git a/frontend/src/components/expert/components/messages/components/ToolCalls.vue b/frontend/src/components/expert/components/messages/components/ToolCalls.vue index 378d02acd4..565d2b7dd7 100644 --- a/frontend/src/components/expert/components/messages/components/ToolCalls.vue +++ b/frontend/src/components/expert/components/messages/components/ToolCalls.vue @@ -10,69 +10,26 @@
-
-
{{ tool.title || tool.name }}
-
- {{ formatKindBadge(tool.kind) }} - {{ tool.name }} -
-
- -
-
- - Input -
-
- -
-
-
- -
-
- - Output - {{ tool.durationMs || 0 }} ms -
-
- -
-
-
-
-
+ :tool="tool" + :expanded="expanded" + />
From 024a47b1738aaaf6165cb5bf29a286f40f71f619 Mon Sep 17 00:00:00 2001 From: cstns Date: Tue, 21 Apr 2026 15:52:34 +0300 Subject: [PATCH 27/54] adds a hidden download chat functionality when pressing ctrl+alt+delete --- .../src/components/expert/components/ExpertMessages.vue | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frontend/src/components/expert/components/ExpertMessages.vue b/frontend/src/components/expert/components/ExpertMessages.vue index bc64a6cded..a746ce7df2 100644 --- a/frontend/src/components/expert/components/ExpertMessages.vue +++ b/frontend/src/components/expert/components/ExpertMessages.vue @@ -22,6 +22,7 @@ import AiMessage from './messages/AiMessage.vue' import HumanMessage from './messages/HumanMessage.vue' import SystemMessage from './messages/SystemMessage.vue' +import { downloadData } from '@/composables/Download.js' import { useProductExpertStore } from '@/stores/product-expert.js' export default { @@ -46,11 +47,19 @@ export default { }, mounted () { this.mountResizeObserver() + window.addEventListener('keydown', this.onKeyDown) }, beforeUnmount () { this.unmountResizeObserver() + window.removeEventListener('keydown', this.onKeyDown) }, methods: { + onKeyDown (e) { + if (e.ctrlKey && e.altKey && e.key === 'd') { + e.preventDefault() + downloadData(this.messages, 'expert-messages.json') + } + }, mountResizeObserver () { const el = this.$refs.messagesWrapper if (!el) return From 6b35885cb72daef4aba452b280289a129bfbf234 Mon Sep 17 00:00:00 2001 From: cstns Date: Tue, 21 Apr 2026 18:01:21 +0300 Subject: [PATCH 28/54] fix race condition when restarting the session --- frontend/src/stores/product-expert.js | 58 ++++----------------------- 1 file changed, 8 insertions(+), 50 deletions(-) diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index e7b77aa738..0b0d60d493 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -1,6 +1,6 @@ import { defineStore } from 'pinia' import { v4 as uuidv4 } from 'uuid' -import { markRaw, watch } from 'vue' +import { markRaw } from 'vue' import expertApi from '../api/expert.js' import userApi from '../api/user.js' @@ -309,6 +309,13 @@ export const useProductExpertStore = defineStore('product-expert', { agentStore.sessionId = uuidv4() agentStore.messages = [] + if (this.shouldUseMqtt) { + const servicesOrchestrator = getServicesOrchestrator() + const mqttService = servicesOrchestrator.$serviceInstances.mqtt + + await mqttService.destroyClient(this.mqttConnectionKey) + } + // Reset session timing this.resetSessionTimer() this.startSessionTimer() @@ -639,55 +646,6 @@ export const useProductExpertStore = defineStore('product-expert', { }), { qos: 2 } ) - - // whenever the sessionId changes, we need to unsubscribe from previous topics and subscribe to the - // new ones based off of the new sessionId - watch( - () => this._agentStore.sessionId, - async () => { - if (!mqttService.hasClient(this.mqttConnectionKey)) return - - const timerHelper = useTimerHelper() - await mqttService.destroyClient(this.mqttConnectionKey) - - // TODO getting required creds fails from time to time because we're creating the client - // before the backend successfully remove the old one - await this.establishMqttComms() - - const managedClient = mqttService.getManagedClient(this.mqttConnectionKey) - - await timerHelper.waitWhile( - () => ['connected'].includes(managedClient.status), - { intervalMs: 200, cutoffTries: 50 } - ) - - await new Promise(resolve => setTimeout(resolve, 5000)) - - await mqttService.subscribe( - this.mqttConnectionKey, - mqttTopicHelper.buildTopic({ - entityType: '+', - entityId: '+', - agentChannel: 'support', - topicType: 'chat', - topicAction: 'response' - }), - { qos: 2 } - ) - await mqttService.subscribe( - this.mqttConnectionKey, - mqttTopicHelper.buildTopic({ - entityType: '+', - entityId: '+', - agentChannel: 'support', - topicType: 'inflight', - topicAction: 'request', - inflightType: '+' - }), - { qos: 2 } - ) - } - ) }, _onMqttOffline () { console.log('#################### mqtt offline') From c6cdf03ecb55f13f1f96938a87c3eaad89e413ac Mon Sep 17 00:00:00 2001 From: cstns Date: Tue, 21 Apr 2026 19:06:55 +0300 Subject: [PATCH 29/54] move the tool call duration in the tool call title --- .../components/messages/components/ToolCallItem.vue | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/expert/components/messages/components/ToolCallItem.vue b/frontend/src/components/expert/components/messages/components/ToolCallItem.vue index 750d08433e..0e75658fee 100644 --- a/frontend/src/components/expert/components/messages/components/ToolCallItem.vue +++ b/frontend/src/components/expert/components/messages/components/ToolCallItem.vue @@ -2,8 +2,11 @@
{{ tool.title || tool.name }}
- {{ formatKindBadge(tool.kind) }} - {{ tool.name }} +
+ {{ formatKindBadge(tool.kind) }} + {{ tool.name }} +
+ {{ tool.durationMs || 0 }} ms
@@ -34,7 +37,6 @@ :class="{ 'rotated': outputExpanded }" /> Output - {{ tool.durationMs || 0 }} ms
@@ -134,6 +136,7 @@ export default { .ff-expert-tool-call--name { display: flex; align-items: center; + justify-content: space-between; gap: 0.375rem; font-size: 0.75rem; color: $ff-grey-500; From 10d2c72db77e7e24b962421aa1415b9f16310551 Mon Sep 17 00:00:00 2001 From: cstns Date: Tue, 21 Apr 2026 19:57:16 +0300 Subject: [PATCH 30/54] remove code block border/padding from agent responses --- .../components/resources/StreamableContent.vue | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/frontend/src/components/expert/components/messages/components/resources/StreamableContent.vue b/frontend/src/components/expert/components/messages/components/resources/StreamableContent.vue index b4d31b638f..43bdb2ab8f 100644 --- a/frontend/src/components/expert/components/messages/components/resources/StreamableContent.vue +++ b/frontend/src/components/expert/components/messages/components/resources/StreamableContent.vue @@ -223,6 +223,12 @@ export default { } } + :deep(code) { + padding: 0; + border: none; + border-radius: 0; + } + :deep(table) { border-collapse: collapse; /* removes double borders */ @@ -233,15 +239,6 @@ export default { td, th { padding: 5px 10px; - - } - - td { - code { - padding: 0; - border: none; - border-radius: 0; - } } } } From 022ce9fc61e694b9a1d6495ab77939460a175717 Mon Sep 17 00:00:00 2001 From: cstns Date: Wed, 22 Apr 2026 11:15:13 +0300 Subject: [PATCH 31/54] qf linting error --- frontend/src/stores/product-expert.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 0b0d60d493..98a0da0ea6 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -648,7 +648,7 @@ export const useProductExpertStore = defineStore('product-expert', { ) }, _onMqttOffline () { - console.log('#################### mqtt offline') + console.warn('#################### mqtt offline') // TODO add error message, handle reconnect, notify user }, _onMqttError (e) { From 6fe946da1996baf45f1d08f06e92291be986462b Mon Sep 17 00:00:00 2001 From: cstns Date: Wed, 22 Apr 2026 11:44:06 +0300 Subject: [PATCH 32/54] rename store props for clarity and fix fe unit tests --- .../expert/components/ExpertChatInput.vue | 6 +-- frontend/src/stores/product-assistant.js | 4 +- frontend/src/stores/product-expert.js | 4 +- .../frontend/stores/product-assistant.spec.js | 42 ++++++++++++------- .../frontend/stores/product-expert.spec.js | 2 +- 5 files changed, 36 insertions(+), 22 deletions(-) diff --git a/frontend/src/components/expert/components/ExpertChatInput.vue b/frontend/src/components/expert/components/ExpertChatInput.vue index 9a0a61ce61..c9310c59a4 100644 --- a/frontend/src/components/expert/components/ExpertChatInput.vue +++ b/frontend/src/components/expert/components/ExpertChatInput.vue @@ -113,8 +113,8 @@ export default { }, computed: { ...mapState(useProductAssistantStore, [ - 'immersiveInstance', - 'immersiveDevice' + 'isImmersiveInstance', + 'isImmersiveDevice' ]), ...mapState(useUxDrawersStore, ['rightDrawer']), ...mapState(useProductExpertStore, [ @@ -145,7 +145,7 @@ export default { : 'Tell us what you need help with' }, isImmersive () { - return this.immersiveDevice || this.immersiveInstance + return this.isImmersiveDevice || this.isImmersiveInstance } }, mounted () { diff --git a/frontend/src/stores/product-assistant.js b/frontend/src/stores/product-assistant.js index 779ec73a27..c9309f6b0c 100644 --- a/frontend/src/stores/product-assistant.js +++ b/frontend/src/stores/product-assistant.js @@ -161,12 +161,12 @@ export const useProductAssistantStore = defineStore('product-assistant', { pendingRequests: new Map() // key is transactionId, value is { resolve, reject, timeout, timestamp, type, action, params } }), getters: { - immersiveInstance: () => { + isImmersiveInstance: () => { const contextStore = useContextStore() return contextStore.instance && contextStore.isImmersive }, - immersiveDevice: () => { + isImmersiveDevice: () => { const contextStore = useContextStore() return contextStore.device && contextStore.isImmersive diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 98a0da0ea6..0d2dfe8309 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -55,11 +55,11 @@ export const useProductExpertStore = defineStore('product-expert', { }, canImportFlows () { const assistantStore = useProductAssistantStore() - return !!assistantStore.immersiveInstance && !!assistantStore.supportedActions['custom:import-flow'] + return !!assistantStore.isImmersiveInstance && !!assistantStore.supportedActions['custom:import-flow'] }, canManagePalette () { const assistantStore = useProductAssistantStore() - return !!assistantStore.immersiveInstance && !!assistantStore.supportedActions['core:manage-palette'] + return !!assistantStore.isImmersiveInstance && !!assistantStore.supportedActions['core:manage-palette'] }, mqttConnectionKey () { return this._agentStore.mqttConnectionKey }, sessionId () { return this._agentStore.sessionId }, diff --git a/test/unit/frontend/stores/product-assistant.spec.js b/test/unit/frontend/stores/product-assistant.spec.js index 68029a55c0..f9c63932e5 100644 --- a/test/unit/frontend/stores/product-assistant.spec.js +++ b/test/unit/frontend/stores/product-assistant.spec.js @@ -86,33 +86,47 @@ describe('product-assistant store', () => { }) describe('getters', () => { - describe('immersiveInstance', () => { - it('returns null when context store has no instance', () => { + describe('isImmersiveInstance', () => { + it('returns falsy when context store has no instance', () => { const store = useProductAssistantStore() - expect(store.immersiveInstance).toBeNull() + expect(store.isImmersiveInstance).toBeFalsy() }) - it('returns the instance from the context store', () => { + it('returns falsy when instance is set but isImmersive is false', () => { const contextStore = useContextStore() const store = useProductAssistantStore() - const mockInstance = { id: 'inst-1', url: 'http://localhost:1880' } - contextStore.setInstance(mockInstance) - expect(store.immersiveInstance).toEqual(mockInstance) + contextStore.setInstance({ id: 'inst-1', url: 'http://localhost:1880' }) + expect(store.isImmersiveInstance).toBeFalsy() + }) + + it('returns true when instance is set and isImmersive is true', () => { + const contextStore = useContextStore() + const store = useProductAssistantStore() + contextStore.setInstance({ id: 'inst-1', url: 'http://localhost:1880' }) + contextStore.setIsImmersive(true) + expect(store.isImmersiveInstance).toBe(true) }) }) - describe('immersiveDevice', () => { - it('returns null when context store has no device', () => { + describe('isImmersiveDevice', () => { + it('returns falsy when context store has no device', () => { + const store = useProductAssistantStore() + expect(store.isImmersiveDevice).toBeFalsy() + }) + + it('returns falsy when device is set but isImmersive is false', () => { + const contextStore = useContextStore() const store = useProductAssistantStore() - expect(store.immersiveDevice).toBeNull() + contextStore.setDevice({ id: 'dev-1', editor: { url: 'http://device.local' } }) + expect(store.isImmersiveDevice).toBeFalsy() }) - it('returns the device from the context store', () => { + it('returns true when device is set and isImmersive is true', () => { const contextStore = useContextStore() const store = useProductAssistantStore() - const mockDevice = { id: 'dev-1', editor: { url: 'http://device.local' } } - contextStore.setDevice(mockDevice) - expect(store.immersiveDevice).toEqual(mockDevice) + contextStore.setDevice({ id: 'dev-1', editor: { url: 'http://device.local' } }) + contextStore.setIsImmersive(true) + expect(store.isImmersiveDevice).toBe(true) }) }) diff --git a/test/unit/frontend/stores/product-expert.spec.js b/test/unit/frontend/stores/product-expert.spec.js index 3ccae5fa9a..59ab6fdc19 100644 --- a/test/unit/frontend/stores/product-expert.spec.js +++ b/test/unit/frontend/stores/product-expert.spec.js @@ -13,7 +13,7 @@ vi.mock('@/stores/context.js', () => ({ vi.mock('@/stores/product-assistant.js', () => ({ useProductAssistantStore: vi.fn(() => ({ - immersiveInstance: null, + isImmersiveInstance: null, supportedActions: {} })) })) From ef2dd51b329bcc19a53fd8f3204159a843df98cd Mon Sep 17 00:00:00 2001 From: cstns Date: Tue, 28 Apr 2026 14:22:48 +0300 Subject: [PATCH 33/54] Refactor in-flight request handling logic to improve code clarity --- frontend/src/stores/product-expert.js | 66 ++++++++++++--------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 0d2dfe8309..563df34729 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -222,27 +222,18 @@ export const useProductExpertStore = defineStore('product-expert', { onError: this._onMqttError }) }, - async handleInFlightRequest ({ topic, message, packet, transactionId, sessionId, chatTransactionId } = {}) { - // console.log('handleInFlightRequest', { - // topic, - // message, - // packet, - // transactionId, - // sessionId, - // chatTransactionId - // }) - + async handleInFlightRequest ({ topic, message, transactionId, sessionId, chatTransactionId } = {}) { const inFlightRequest = this._inFlightRequests.values().next().value + + // dismiss inFlight requests that don't match the existing sessionId or the inFlight message transactionId if (sessionId !== this.sessionId || inFlightRequest?.transactionId !== chatTransactionId) return const servicesOrchestrator = getServicesOrchestrator() - const mqttService = servicesOrchestrator.$serviceInstances.mqtt - const assistantStore = useProductAssistantStore() - const topicHelper = useMqttExpertTopicHelper() - const parsedTopic = topicHelper.parseTopic(topic) + const mqttService = servicesOrchestrator.$serviceInstances.mqtt + const parsedTopic = topicHelper.parseTopic(topic) const payload = JSON.parse(message.toString()) this._addInFlightUpdate(payload.status || payload.toolname || 'Processing request...') @@ -254,13 +245,11 @@ export const useProductExpertStore = defineStore('product-expert', { topicType: 'inflight', topicAction: 'response', inflightType: parsedTopic.inflightType, - sessionId: sessionId || this.sessionId // TODO: consider validating the incoming request session in userProps matches + sessionId: sessionId || this.sessionId }) - if (parsedTopic.inflightType === 'expert:status-message') { - // this is just a status from the agent, we can ignore it for now, but we might want to display it in the UI later - // just ack the msg - // TODO: We should be updating the message in the chat interface with _busy doing xyz_ status updates instead of the made up text like "Reading Rag", "Calling MCP" etc. + switch (true) { + case parsedTopic.inflightType === 'expert:status-message': await mqttService.publishMessage(this.mqttConnectionKey, { qos: 2, topic: responseTopic, @@ -268,31 +257,32 @@ export const useProductExpertStore = defineStore('product-expert', { ack: true }), correlationData: new TextEncoder().encode(transactionId), - userProperties: { sessionId, transactionId: chatTransactionId } // pass through for integrity, not actually needed for correlation in this case since it's just an ack + userProperties: { sessionId, transactionId: chatTransactionId } }) - } else if (parsedTopic.inflightType.startsWith('automation:')) { - // thi is an automation request, we want to execute it and return the result to the agent - const actionName = parsedTopic.inflightType.replace('automation:', '') - const postMessagePayload = { - action: `automation/${actionName}`, - params: payload.params || {}, - sessionId, // the chat session - chatTransactionId, // the chat transaction, passed through for integrity - transactionId // the incoming MQTT transaction (not the original chat transaction - we store that in userProperties) - } - + break + case parsedTopic.inflightType.startsWith('automation:'): try { - const result = await assistantStore.invokeActionAwaitResponse(postMessagePayload) + const result = await assistantStore.invokeActionAwaitResponse({ + action: `automation/${parsedTopic.inflightType.replace('automation:', '')}`, + params: payload.params || {}, + sessionId, + chatTransactionId, + transactionId + }) + await mqttService.publishMessage(this.mqttConnectionKey, { qos: 2, topic: responseTopic, payload: JSON.stringify(result), correlationData: new TextEncoder().encode(transactionId), - userProperties: { sessionId, transactionId: chatTransactionId } // pass through for integrity + userProperties: { sessionId, transactionId: chatTransactionId } }) } catch (e) { this._onMqttError(e) } + break + default: + // do nothing } }, async handleMessageResponse (response) { @@ -593,13 +583,14 @@ export const useProductExpertStore = defineStore('product-expert', { const sessionId = packet.properties?.userProperties?.sessionId // chat session const chatTransactionId = packet.properties?.userProperties?.transactionId // passed through for integrity - if (parsedTopic.isReply) { + switch (true) { + case parsedTopic.isReply: // remove inFlight request because it is now resolved this._inFlightRequests.delete(transactionId) - // handle the response await this.handleMessageResponse(JSON.parse(message.toString())) - } else if (parsedTopic.isInflightRequest) { + break + case parsedTopic.isInflightRequest: await this.handleInFlightRequest({ topic, message, @@ -608,6 +599,9 @@ export const useProductExpertStore = defineStore('product-expert', { sessionId, chatTransactionId }) + break + default: + // do nothing } }, _onMqttClose () { From 4d72847b39ac65a41bad82f5dd15611053982068 Mon Sep 17 00:00:00 2001 From: cstns Date: Tue, 28 Apr 2026 14:32:25 +0300 Subject: [PATCH 34/54] Simplify message handling logic and improve readability in hydrateMessages because we're not receiving tool cals when ingestign the context from the website --- frontend/src/stores/product-expert.js | 58 +++++---------------------- 1 file changed, 11 insertions(+), 47 deletions(-) diff --git a/frontend/src/stores/product-expert.js b/frontend/src/stores/product-expert.js index 563df34729..ab75e98f16 100644 --- a/frontend/src/stores/product-expert.js +++ b/frontend/src/stores/product-expert.js @@ -521,56 +521,20 @@ export const useProductExpertStore = defineStore('product-expert', { }, hydrateMessages (messages) { if (!messages) return - messages.forEach((message) => { - // TODO break this into manageable chunks, do we actually still need it? - if (message.answer && Array.isArray(message.answer)) { - // Extract MCP items (tools, resources, resource templates, prompts) from the answer array - const mcpItems = message.answer.filter(item => - item.kind === 'mcp_tool' || - item.kind === 'mcp_resource' || - item.kind === 'mcp_resource_template' || - item.kind === 'mcp_prompt' - ) - - // Handle MCP calls if present - includes tools, resources, and prompts - if (mcpItems.length > 0) { - const toolCalls = mcpItems.map(item => ({ - id: item.toolId, - name: item.toolName, - title: item.toolTitle || item.toolName, - kind: item.kind, - args: item.input, - output: item.output, - durationMs: item.durationMs - })) - - // Calculate total duration in seconds - const totalDurationMs = mcpItems.reduce((sum, item) => sum + (item.durationMs || 0), 0) - const totalDurationSec = (totalDurationMs / 1000).toFixed(2) - - this._agentStore.messages.push({ - _type: 'ai', - kind: 'tool_calls', - toolCalls, - duration: totalDurationSec, - content: `${toolCalls.length} tool call(s)`, - _timestamp: Date.now(), - _streamed: true, - _uuid: uuidv4() - }) - } + const isAiMessage = (message) => message.answer && Array.isArray(message.answer) + const isUserMessage = (message) => Object.prototype.hasOwnProperty.call(message, 'query') && message.query + messages.forEach((message) => { + switch (true) { + case isAiMessage(message): this.addAiMessage(message, false) - } else if (message.query) { - // Transform user message - this._agentStore.messages.push({ - _type: 'human', - content: message.query, - _timestamp: Date.now(), - _uuid: uuidv4() - }) + break + case isUserMessage(message): + this.addUserMessage(message.query) + break + default: + // do nothing, unrecognized message } - // Else: ignore messages that don't match either format }) }, async _onMqttMessage (topic, message, packet) { From 5151312b0ee24e7682ca63444b91d8ddcab8e36b Mon Sep 17 00:00:00 2001 From: cstns Date: Wed, 29 Apr 2026 17:17:12 +0300 Subject: [PATCH 35/54] Adjust keyboard shortcut modifier detection for cross-platform compatibility --- .../src/components/expert/components/ExpertMessages.vue | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/expert/components/ExpertMessages.vue b/frontend/src/components/expert/components/ExpertMessages.vue index a746ce7df2..1b78d189fe 100644 --- a/frontend/src/components/expert/components/ExpertMessages.vue +++ b/frontend/src/components/expert/components/ExpertMessages.vue @@ -55,7 +55,11 @@ export default { }, methods: { onKeyDown (e) { - if (e.ctrlKey && e.altKey && e.key === 'd') { + const platform = navigator.userAgentData?.platform || navigator.platform || '' + const isMac = /mac/i.test(platform) + const primaryModifier = isMac ? e.metaKey : e.ctrlKey + + if (primaryModifier && e.altKey && e.key.toLowerCase() === 'd') { e.preventDefault() downloadData(this.messages, 'expert-messages.json') } From cd8e0d13081ffafc02cd135efd09f2c7dd4b5831 Mon Sep 17 00:00:00 2001 From: cstns Date: Wed, 29 Apr 2026 17:37:00 +0300 Subject: [PATCH 36/54] set contextual device when in immersive --- frontend/src/pages/device/Editor/index.vue | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/pages/device/Editor/index.vue b/frontend/src/pages/device/Editor/index.vue index 28516dd213..0d97317341 100644 --- a/frontend/src/pages/device/Editor/index.vue +++ b/frontend/src/pages/device/Editor/index.vue @@ -278,6 +278,7 @@ export default { handler (device) { if (device && this.isEditorAvailable) { this.bindDevice(device, true) + this.setContextualDevice(device) this.handlePolling() } else { this.$router.push({ name: 'device-overview' }) @@ -319,7 +320,10 @@ export default { this.stopPolling() }, methods: { - ...mapActions(useContextStore, ['setIsImmersive']), + ...mapActions(useContextStore, { + setIsImmersive: 'setIsImmersive', + setContextualDevice: 'setDevice' + }), loadDevice: async function () { let tries = 0 let device = await this.fetchDevice(this.$route.params.id, false) From 4891a18f26bed568b558b8aedddf39836d6a5401 Mon Sep 17 00:00:00 2001 From: cstns Date: Wed, 29 Apr 2026 18:54:34 +0300 Subject: [PATCH 37/54] Adjust keyboard shortcut to use Shift + Alt + d for consistency across os's when downloading chat messages --- .../src/components/expert/components/ExpertMessages.vue | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/frontend/src/components/expert/components/ExpertMessages.vue b/frontend/src/components/expert/components/ExpertMessages.vue index 1b78d189fe..76ffe3b574 100644 --- a/frontend/src/components/expert/components/ExpertMessages.vue +++ b/frontend/src/components/expert/components/ExpertMessages.vue @@ -55,11 +55,7 @@ export default { }, methods: { onKeyDown (e) { - const platform = navigator.userAgentData?.platform || navigator.platform || '' - const isMac = /mac/i.test(platform) - const primaryModifier = isMac ? e.metaKey : e.ctrlKey - - if (primaryModifier && e.altKey && e.key.toLowerCase() === 'd') { + if (e.altKey && e.shiftKey && e.key.toLowerCase() === 'd') { e.preventDefault() downloadData(this.messages, 'expert-messages.json') } From 9fc0536bd70a44f5d47b63023fbda31a2f2cadda Mon Sep 17 00:00:00 2001 From: cstns Date: Thu, 30 Apr 2026 10:33:14 +0300 Subject: [PATCH 38/54] qf expert chat selection pills overflowing input --- .../src/components/expert/components/ExpertChatInput.vue | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/expert/components/ExpertChatInput.vue b/frontend/src/components/expert/components/ExpertChatInput.vue index c9310c59a4..9238db99f2 100644 --- a/frontend/src/components/expert/components/ExpertChatInput.vue +++ b/frontend/src/components/expert/components/ExpertChatInput.vue @@ -33,7 +33,7 @@ />
-
+
@@ -64,13 +64,13 @@