Fix analytics collector and ai-bot services

.vscode/launch.json

@@ -347,28 +347,6 @@
       "cwd": "${workspaceRoot}/services/rekoni",
       "protocol": "inspector"
     },
-    {
-      "name": "Debug AI bot",
-      "type": "node",
-      "request": "launch",
-      "args": ["src/index.ts"],
-      "env": {
-        "ACCOUNTS_URL": "http://localhost:3000",
-        "MONGO_URL": "mongodb://localhost:27017",
-        "PORT": "4008",
-        "SERVER_SECRET": "secret",
-        "SUPPORT_WORKSPACE": "support",
-        "FIRST_NAME": "Jolie",
-        "LAST_NAME": "AI",
-        "AVATAR_PATH": "./assets/avatar.png",
-        "AVATAR_CONTENT_TYPE": ".png"
-      },
-      "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
-      "sourceMaps": true,
-      "cwd": "${workspaceRoot}/services/ai-bot/pod-ai-bot",
-      "protocol": "inspector",
-      "outputCapture": "std"
-    },
     {
       "name": "Debug analytics collector",
       "type": "node",

dev/prod/config.json

@@ -5,5 +5,6 @@
   "UPLOAD_URL":"/files",
   "REKONI_URL": "http://localhost:4004",
   "PRINT_URL": "http://localhost:4005",
-  "SIGN_URL": "http://localhost:4006"
+  "SIGN_URL": "http://localhost:4006",
+  "ANALYTICS_COLLECTOR_URL":"http://localhost:4007"
 }

models/all/src/migration.ts

@@ -50,6 +50,7 @@ import { trainingOperation } from '@hcengineering/model-training'
 import { documentsOperation } from '@hcengineering/model-controlled-documents'
 import { productsOperation } from '@hcengineering/model-products'
 import { requestOperation } from '@hcengineering/model-request'
+import { analyticsCollectorOperation } from '@hcengineering/model-analytics-collector'
 
 export const migrateOperations: [string, MigrateOperation][] = [
   ['core', coreOperation],
@@ -88,5 +89,6 @@ export const migrateOperations: [string, MigrateOperation][] = [
   ['activityServer', activityServerOperation],
   ['textEditorOperation', textEditorOperation],
   // We should call notification migration after activityServer and chunter
-  ['notification', notificationOperation]
+  ['notification', notificationOperation],
+  ['analyticsCollector', analyticsCollectorOperation]
 ]

models/analytics-collector/package.json

@@ -32,6 +32,8 @@
     "@hcengineering/chunter": "^0.6.20",
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/model": "^0.6.11",
+    "@hcengineering/model-activity": "^0.6.0",
+    "@hcengineering/model-notification": "^0.6.0",
     "@hcengineering/model-chunter": "^0.6.0",
     "@hcengineering/model-core": "^0.6.0",
     "@hcengineering/model-view": "^0.6.0",

models/analytics-collector/src/index.ts

@@ -21,6 +21,7 @@ import { TChannel } from '@hcengineering/model-chunter'
 import analyticsCollector from './plugin'
 
 export { analyticsCollectorId } from '@hcengineering/analytics-collector'
+export { analyticsCollectorOperation } from './migration'
 export default analyticsCollector
 
 @Mixin(analyticsCollector.mixin.AnalyticsChannel, chunter.class.Channel)

models/analytics-collector/src/migration.ts

@@ -0,0 +1,56 @@
+//
+// Copyright © 2020, 2021 Anticrm Platform Contributors.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import {
+  type MigrateOperation,
+  type MigrationClient,
+  type MigrationUpgradeClient,
+  tryMigrate
+} from '@hcengineering/model'
+import analyticsCollector, { analyticsCollectorId } from '@hcengineering/analytics-collector'
+import { DOMAIN_SPACE } from '@hcengineering/model-core'
+import { DOMAIN_DOC_NOTIFY, DOMAIN_NOTIFICATION } from '@hcengineering/model-notification'
+import { DOMAIN_ACTIVITY } from '@hcengineering/model-activity'
+
+async function removeAnalyticsChannels (client: MigrationClient): Promise<void> {
+  const channels = await client.find(DOMAIN_SPACE, {
+    [`${analyticsCollector.mixin.AnalyticsChannel}`]: { $exists: true }
+  })
+
+  if (channels.length === 0) {
+    return
+  }
+
+  const channelsIds = channels.map((it) => it._id)
+  const contexts = await client.find(DOMAIN_DOC_NOTIFY, { objectId: { $in: channelsIds } })
+  const contextsIds = contexts.map((it) => it._id)
+
+  await client.deleteMany(DOMAIN_ACTIVITY, { attachedTo: { $in: channelsIds } })
+  await client.deleteMany(DOMAIN_NOTIFICATION, { docNotifyContext: { $in: contextsIds } })
+  await client.deleteMany(DOMAIN_DOC_NOTIFY, { _id: { $in: contextsIds } })
+  await client.deleteMany(DOMAIN_SPACE, { [`${analyticsCollector.mixin.AnalyticsChannel}`]: { $exists: true } })
+}
+
+export const analyticsCollectorOperation: MigrateOperation = {
+  async migrate (client: MigrationClient): Promise<void> {
+    await tryMigrate(client, analyticsCollectorId, [
+      {
+        state: 'remove-analytics-channels-v1',
+        func: removeAnalyticsChannels
+      }
+    ])
+  },
+  async upgrade (state: Map<string, Set<string>>, client: () => Promise<MigrationUpgradeClient>): Promise<void> {}
+}

models/server-ai-bot/src/index.ts

@@ -65,6 +65,7 @@ export class TAIBotTransferEvent extends TAIBotEvent implements AIBotTransferEve
   toEmail!: string
   toWorkspace!: string
   fromWorkspace!: string
+  fromWorkspaceUrl!: string
   messageId!: Ref<ChatMessage>
   parentMessageId?: Ref<ChatMessage>
 }

plugins/ai-bot/src/index.ts

@@ -39,6 +39,7 @@ export interface AIBotTransferEvent extends AIBotEvent {
   toEmail: string
   toWorkspace: string
   fromWorkspace: string
+  fromWorkspaceUrl: string
   messageId: Ref<ChatMessage>
   parentMessageId?: Ref<ChatMessage>
 }

plugins/chunter-resources/src/channelDataProvider.ts

@@ -244,7 +244,7 @@ export class ChannelDataProvider implements IChannelDataProvider {
       this.isTailLoading.set(true)
       const tailStart = metadata[startIndex]?.createdOn
       this.loadTail(tailStart)
-      this.backwardNextPromise = this.loadNext('backward', metadata[startIndex]?.createdOn, this.limit)
+      this.backwardNextPromise = this.loadNext('backward', metadata[startIndex]?.createdOn, this.limit, false)
     } else {
       const newStart = Math.max(startPosition - this.limit / 2, 0)
       await this.loadMore('forward', metadata[newStart]?.createdOn, this.limit)
@@ -309,7 +309,7 @@ export class ChannelDataProvider implements IChannelDataProvider {
     return index !== -1 ? metadata.length - index : -1
   }
 
-  async loadChunk (isBackward: boolean, loadAfter: Timestamp, limit?: number): Promise<Chunk | undefined> {
+  async loadChunk (isBackward: boolean, loadAfter: Timestamp, limit?: number, equal = true): Promise<Chunk | undefined> {
     const client = getClient()
     const skipIds = this.getChunkSkipIds(loadAfter)
 
@@ -319,7 +319,13 @@ export class ChannelDataProvider implements IChannelDataProvider {
         attachedTo: this.chatId,
         space: this.space,
         _id: { $nin: skipIds },
-        createdOn: isBackward ? { $lte: loadAfter } : { $gte: loadAfter }
+        createdOn: equal
+          ? isBackward
+            ? { $lte: loadAfter }
+            : { $gte: loadAfter }
+          : isBackward
+            ? { $lt: loadAfter }
+            : { $gt: loadAfter }
       },
       {
         limit: limit ?? this.limit,
@@ -359,7 +365,7 @@ export class ChannelDataProvider implements IChannelDataProvider {
       .map(({ _id }) => _id)
   }
 
-  async loadNext (mode: LoadMode, loadAfter?: Timestamp, limit?: number): Promise<void> {
+  async loadNext (mode: LoadMode, loadAfter?: Timestamp, limit?: number, equal = true): Promise<void> {
     if (this.chatId === undefined || loadAfter === undefined) {
       return
     }
@@ -384,7 +390,7 @@ export class ChannelDataProvider implements IChannelDataProvider {
       return
     }
 
-    const chunk = await this.loadChunk(isBackward, loadAfter, limit)
+    const chunk = await this.loadChunk(isBackward, loadAfter, limit, equal)
 
     if (chunk !== undefined && isBackward) {
       this.backwardNextStore.set(chunk)

plugins/chunter-resources/src/components/ChannelScrollView.svelte

@@ -25,7 +25,7 @@
     canGroupMessages,
     messageInFocus
   } from '@hcengineering/activity-resources'
-  import { Class, Doc, getDay, Ref, Timestamp } from '@hcengineering/core'
+  import { Class, Doc, generateId, getDay, Ref, Timestamp } from '@hcengineering/core'
   import { InboxNotificationsClientImpl } from '@hcengineering/notification-resources'
   import { getResource } from '@hcengineering/platform'
   import { getClient } from '@hcengineering/presentation'
@@ -657,29 +657,32 @@
     }
   }
 
-  function handleScrollDown (): void {
+  async function handleScrollDown (): Promise<void> {
     selectedMessageId = undefined
     messageInFocus.set(undefined)
 
     const metadata = $metadataStore
     const lastMetadata = metadata[metadata.length - 1]
     const lastMessage = displayMessages[displayMessages.length - 1]
 
-    void inboxClient.readDoc(client, objectId)
-
     if (lastMetadata._id !== lastMessage._id) {
       separatorIndex = -1
       provider.jumpToEnd(true)
       reinitializeScroll()
     } else {
       scrollToBottom()
     }
+
+    const op = client.apply(generateId(), 'chunter.scrollDown')
+    await inboxClient.readDoc(op, objectId)
+    await op.commit()
   }
 
-  $: forceReadContext(isScrollAtBottom, notifyContext)
+  let forceRead = false
+  $: void forceReadContext(isScrollAtBottom, notifyContext)
 
-  function forceReadContext (isScrollAtBottom: boolean, context?: DocNotifyContext): void {
-    if (context === undefined || !isScrollAtBottom) return
+  async function forceReadContext (isScrollAtBottom: boolean, context?: DocNotifyContext): Promise<void> {
+    if (context === undefined || !isScrollAtBottom || forceRead || !separatorElement) return
     const { lastUpdateTimestamp = 0, lastViewedTimestamp = 0 } = context
 
     if (lastViewedTimestamp >= lastUpdateTimestamp) return
@@ -688,7 +691,10 @@
     const unViewed = notifications.filter(({ isViewed }) => !isViewed)
 
     if (unViewed.length === 0) {
-      void inboxClient.readDoc(client, objectId)
+      forceRead = true
+      const op = client.apply(generateId(), 'chunter.forceReadContext')
+      await inboxClient.readDoc(op, objectId)
+      await op.commit()
     }
   }
 

plugins/chunter-resources/src/utils.ts

@@ -417,7 +417,7 @@ export function recheckNotifications (context: DocNotifyContext): void {
     const toReadData = Array.from(toRead)
     toRead.clear()
     void (async () => {
-      const _client = client.apply(generateId())
+      const _client = client.apply(generateId(), 'recheckNotifications')
       await inboxClient.readNotifications(_client, toReadData)
       await _client.commit()
     })()
@@ -434,7 +434,7 @@ export async function readChannelMessages (
 
   const inboxClient = InboxNotificationsClientImpl.getClient()
 
-  const client = getClient().apply(generateId())
+  const client = getClient().apply(generateId(), 'readViewportMessages')
   try {
     const readMessages = get(chatReadMessagesStore)
     const allIds = getAllIds(messages).filter((id) => !readMessages.has(id))

plugins/notification-resources/src/inboxNotificationsClient.ts

@@ -161,12 +161,14 @@ export class InboxNotificationsClientImpl implements InboxNotificationsClient {
       return
     }
 
-    const inboxNotifications = (get(this.inboxNotifications) ?? []).filter(
-      (notification) => notification.docNotifyContext === docNotifyContext._id && !notification.isViewed
+    const inboxNotifications = await client.findAll(
+      notification.class.InboxNotification,
+      { docNotifyContext: docNotifyContext._id, isViewed: false },
+      { projection: { _id: 1, _class: 1, space: 1 } }
     )
 
     for (const notification of inboxNotifications) {
-      await client.update(notification, { isViewed: true })
+      await client.updateDoc(notification._class, notification.space, notification._id, { isViewed: true })
     }
     await client.update(docNotifyContext, { lastViewedTimestamp: Date.now() })
   }

plugins/view-resources/src/middleware.ts

@@ -308,6 +308,7 @@ export class AnalyticsMiddleware extends BasePresentationMiddleware implements P
       }
       if (TxProcessor.isExtendsCUD(etx._class)) {
         const cud = etx as TxCUD<Doc>
+        if (cud.objectClass === core.class.BenchmarkDoc) continue
         const _class = this.client.getHierarchy().getClass(cud.objectClass)
         if (_class.label !== undefined) {
           const label = await translate(_class.label, {}, 'en')