Chore: Move ddp-streamer micro service to its own sub-repo (#25246)

.github/workflows/build_and_test.yml

@@ -368,7 +368,7 @@ jobs:
         run: yarn
 
       - name: Build micro services
-        run: yarn build:services
+        run: yarn build
 
       - name: E2E Test API
         env:
@@ -617,14 +617,14 @@ jobs:
           IMAGE_TAG: check
         run: |
           yarn
-          yarn build:services
+          yarn build
 
           echo "Building Docker image for service: ${{ matrix.service }}:${IMAGE_TAG}"
 
           docker build \
             --build-arg SERVICE=${{ matrix.service }} \
             -t rocketchat/${{ matrix.service }}-service:${IMAGE_TAG} \
-            -f ./apps/meteor/ee/server/services/Dockerfile \
+            -f ./ee/apps/ddp-streamer/Dockerfile \
             .
 
   release-versions:
@@ -872,14 +872,20 @@ jobs:
 
           # first install repo dependencies
           yarn
-          yarn build:services
+          yarn build
 
           echo "Building Docker image for service: ${{ matrix.service }}:${IMAGE_TAG}"
 
+          if [[ "${{ matrix.service }}" == "ddp-streamer" ]]; then
+            DOCKERFILE_PATH="./ee/apps/ddp-streamer/Dockerfile"
+          else
+            DOCKERFILE_PATH="./apps/meteor/ee/server/services/Dockerfile"
+          fi
+
           docker build \
             --build-arg SERVICE=${{ matrix.service }} \
             -t rocketchat/${{ matrix.service }}-service:${IMAGE_TAG} \
-            -f ./apps/meteor/ee/server/services/Dockerfile \
+            -f ${DOCKERFILE_PATH} \
             .
 
           docker push rocketchat/${{ matrix.service }}-service:${IMAGE_TAG}

LICENSE

@@ -2,8 +2,9 @@ Copyright (c) 2015-2022 Rocket.Chat Technologies Corp.
 
 Portions of this software are licensed as follows:
 
-* All content that resides under the "apps/meteor/ee/" directory of this repository, if
-  that directory exists, is licensed under the license defined in "apps/meteor/ee/LICENSE".
+* All content that resides under the "apps/meteor/ee/" and "ee/" directories
+  of this repository, if that directory exists, is licensed under the license
+  defined in "apps/meteor/ee/LICENSE".
 * All third-party components incorporated into the Rocket.Chat Software are
   licensed under the original license provided by the owner of the applicable
   component.

apps/meteor/.scripts/start.js

@@ -69,6 +69,7 @@ function killAllProcesses(mainExitCode) {
 }
 
 function startProcess(opts) {
+	console.log('Starting process', opts.name, opts.command, opts.params, opts.options.cwd);
 	const proc = spawn(opts.command, opts.params, opts.options);
 	processes.push(proc);
 
@@ -125,20 +126,25 @@ function startRocketChat() {
 }
 
 async function startMicroservices() {
+	const waitStart = (resolve) => (message) => {
+		if (message.toString().match('NetworkBroker started successfully')) {
+			return resolve();
+		}
+	};
 	const startService = (name) => {
 		return new Promise((resolve) => {
-			const waitStart = (message) => {
-				if (message.toString().match('NetworkBroker started successfully')) {
-					return resolve();
-				}
-			};
+			const cwd =
+				name === 'ddp-streamer'
+					? path.resolve(srcDir, '..', '..', 'ee', 'apps', name, 'dist', 'ee', 'apps', name)
+					: path.resolve(srcDir, 'ee', 'server', 'services', 'dist', 'ee', 'server', 'services', name);
+
 			startProcess({
 				name: `${name} service`,
 				command: 'node',
-				params: ['service.js'],
-				onData: waitStart,
+				params: [name === 'ddp-streamer' ? 'src/service.js' : 'service.js'],
+				onData: waitStart(resolve),
 				options: {
-					cwd: path.resolve(srcDir, 'ee', 'server', 'services', 'dist', 'ee', 'server', 'services', name),
+					cwd,
 					env: {
 						...appOptions.env,
 						...process.env,

apps/meteor/server/sdk/types/IVoipService.ts

@@ -1,7 +1,7 @@
 import { IVoipCallServerConfig, IVoipManagementServerConfig, ServerType, IRegistrationInfo } from '@rocket.chat/core-typings';
 import type { IVoipConnectorResult, IManagementServerConnectionStatus } from '@rocket.chat/core-typings';
 
-import { CommandHandler } from '../../services/voip/connector/asterisk/CommandHandler';
+import type { CommandHandler } from '../../services/voip/connector/asterisk/CommandHandler';
 
 export interface IVoipService {
 	getServerConfigData(serverType: ServerType): IVoipCallServerConfig | IVoipManagementServerConfig;

ee/apps/ddp-streamer/.eslintrc

@@ -0,0 +1,16 @@
+{
+	"extends": ["@rocket.chat/eslint-config"],
+	"overrides": [
+		{
+			"files": ["**/*.spec.js", "**/*.spec.jsx"],
+			"env": {
+				"jest": true
+			}
+		}
+	],
+	"ignorePatterns": ["**/dist"],
+	"plugins": ["jest"],
+	"env": {
+		"jest/globals": true
+	}
+}

ee/apps/ddp-streamer/Dockerfile

@@ -0,0 +1,32 @@
+FROM node:14.18.3-alpine
+
+ARG SERVICE
+
+WORKDIR /app
+
+COPY ./packages/core-typings/package.json packages/core-typings/package.json
+COPY ./packages/core-typings/dist packages/core-typings/dist
+COPY ./packages/rest-typings/package.json packages/rest-typings/package.json
+COPY ./packages/rest-typings/dist packages/rest-typings/dist
+COPY ./packages/ui-contexts/package.json packages/ui-contexts/package.json
+COPY ./packages/ui-contexts/dist packages/ui-contexts/dist
+
+COPY ./ee/apps/${SERVICE}/dist .
+
+COPY ./package.json .
+COPY ./yarn.lock .
+COPY ./.yarnrc.yml .
+COPY ./.yarn/plugins .yarn/plugins
+COPY ./.yarn/releases .yarn/releases
+COPY ./ee/apps/${SERVICE}/package.json ee/apps/${SERVICE}/package.json
+
+ENV NODE_ENV=production \
+    PORT=3000
+
+WORKDIR /app/ee/apps/${SERVICE}
+
+RUN yarn workspaces focus --production
+
+EXPOSE 3000 9458
+
+CMD ["node", "src/service.js"]

ee/apps/ddp-streamer/package.json

@@ -0,0 +1,50 @@
+{
+	"name": "@rocket.chat/ddp-streamer",
+	"private": true,
+	"version": "0.0.1",
+	"description": "Rocket.Chat DDP-Streamer service",
+	"scripts": {
+		"build": "tsc -p tsconfig.json",
+		"test": "echo \"Error: no test specified\" && exit 1"
+	},
+	"keywords": [
+		"rocketchat"
+	],
+	"author": "Rocket.Chat",
+	"dependencies": {
+		"@rocket.chat/apps-engine": "^1.31.0",
+		"@rocket.chat/core-typings": "workspace:^",
+		"@rocket.chat/emitter": "~0.31.9",
+		"@rocket.chat/rest-typings": "workspace:^",
+		"@rocket.chat/string-helpers": "~0.31.9",
+		"@rocket.chat/ui-contexts": "workspace:^",
+		"colorette": "^1.3.0",
+		"ejson": "^2.2.2",
+		"eventemitter3": "^4.0.7",
+		"fibers": "^5.0.1",
+		"jaeger-client": "^3.19.0",
+		"moleculer": "^0.14.19",
+		"mongodb": "^3.6.10",
+		"nats": "^2.4.0",
+		"pino": "^7.9.2",
+		"sharp": "^0.30.4",
+		"uuid": "^7.0.3",
+		"ws": "^8.5.0"
+	},
+	"devDependencies": {
+		"@rocket.chat/eslint-config": "workspace:^",
+		"@types/ejson": "^2.1.3",
+		"@types/meteor": "1.4.81",
+		"@types/mongodb": "^3.6.20",
+		"@types/node": "^14.18.12",
+		"@types/sharp": "^0.30.2",
+		"@types/uuid": "^8.3.4",
+		"@types/ws": "^8.5.3",
+		"pino-pretty": "^7.6.0",
+		"typescript": "~4.3.5"
+	},
+	"main": "./dist/service.js",
+	"files": [
+		"/dist"
+	]
+}

apps/meteor/ee/server/services/ddp-streamer/DDPStreamer.ts → ee/apps/ddp-streamer/src/DDPStreamer.ts

@@ -3,15 +3,15 @@ import url from 'url';
 
 import WebSocket from 'ws';
 
-import { ListenersModule } from '../../../../server/modules/listeners/listeners.module';
-import { StreamerCentral } from '../../../../server/modules/streamer/streamer.module';
-import { MeteorService } from '../../../../server/sdk';
-import { ServiceClass } from '../../../../server/sdk/types/ServiceClass';
+import { ListenersModule } from '../../../../apps/meteor/server/modules/listeners/listeners.module';
+import { StreamerCentral } from '../../../../apps/meteor/server/modules/streamer/streamer.module';
+import { MeteorService } from '../../../../apps/meteor/server/sdk';
+import { ServiceClass } from '../../../../apps/meteor/server/sdk/types/ServiceClass';
 import { Client } from './Client';
 import { events, server } from './configureServer';
 import { DDP_EVENTS } from './constants';
 import { Autoupdate } from './lib/Autoupdate';
-import notifications from './streams';
+import { notifications } from './streams';
 
 const { PORT: port = 4000 } = process.env;
 

apps/meteor/ee/server/services/ddp-streamer/Server.ts → ee/apps/ddp-streamer/src/Server.ts

@@ -8,9 +8,9 @@ import { DDP_EVENTS } from './constants';
 import { Publication } from './Publication';
 import { Client } from './Client';
 import { IPacket } from './types/IPacket';
-import { MeteorService } from '../../../../server/sdk';
-import { isMeteorError, MeteorError } from '../../../../server/sdk/errors';
-import { Logger } from '../../../../server/lib/logger/Logger';
+import { MeteorService } from '../../../../apps/meteor/server/sdk';
+import { isMeteorError, MeteorError } from '../../../../apps/meteor/server/sdk/errors';
+import { Logger } from '../../../../apps/meteor/server/lib/logger/Logger';
 
 const logger = new Logger('DDP-Streamer');
 

apps/meteor/ee/server/services/ddp-streamer/Streamer.ts → ee/apps/ddp-streamer/src/Streamer.ts

@@ -4,8 +4,8 @@ import type { DDPSubscription, Connection, TransformMessage } from 'meteor/rocke
 import { server } from './configureServer';
 import { DDP_EVENTS } from './constants';
 import { isEmpty } from './lib/utils';
-import { Streamer, StreamerCentral } from '../../../../server/modules/streamer/streamer.module';
-import { api } from '../../../../server/sdk/api';
+import { Streamer, StreamerCentral } from '../../../../apps/meteor/server/modules/streamer/streamer.module';
+import { api } from '../../../../apps/meteor/server/sdk/api';
 
 StreamerCentral.on('broadcast', (name, eventName, args) => {
 	api.broadcast('stream', [name, eventName, args]);

apps/meteor/ee/server/services/ddp-streamer/configureServer.ts → ee/apps/ddp-streamer/src/configureServer.ts

@@ -3,10 +3,10 @@ import { EventEmitter } from 'events';
 import { UserStatus } from '@rocket.chat/core-typings';
 
 import { DDP_EVENTS, WS_ERRORS } from './constants';
-import { Account, Presence, MeteorService } from '../../../../server/sdk';
+import { Account, Presence, MeteorService } from '../../../../apps/meteor/server/sdk';
 import { Server } from './Server';
-import { api } from '../../../../server/sdk/api';
-import { MeteorError } from '../../../../server/sdk/errors';
+import { api } from '../../../../apps/meteor/server/sdk/api';
+import { MeteorError } from '../../../../apps/meteor/server/sdk/errors';
 import { Autoupdate } from './lib/Autoupdate';
 
 export const server = new Server();

apps/meteor/ee/server/services/ddp-streamer/lib/Autoupdate.ts → ee/apps/ddp-streamer/src/lib/Autoupdate.ts

@@ -1,8 +1,8 @@
 import { EventEmitter } from 'events';
 
-import { AutoUpdateRecord } from '../../../../../server/sdk/types/IMeteor';
+import { AutoUpdateRecord } from '../../../../../apps/meteor/server/sdk/types/IMeteor';
 
-export const Autoupdate = new (class Autoupdate extends EventEmitter {
+class AutoupdateSingleton extends EventEmitter {
 	private versions = new Map<string, Omit<AutoUpdateRecord, '_id'>>();
 
 	public updateVersion(record: AutoUpdateRecord): void {
@@ -15,4 +15,6 @@ export const Autoupdate = new (class Autoupdate extends EventEmitter {
 	public getVersions(): Map<string, Omit<AutoUpdateRecord, '_id'>> {
 		return this.versions;
 	}
-})();
+}
+
+export const Autoupdate = new AutoupdateSingleton();

ee/apps/ddp-streamer/src/service.ts

@@ -0,0 +1,6 @@
+import '../../../../apps/meteor/ee/server/startup/broker';
+
+import { api } from '../../../../apps/meteor/server/sdk/api';
+import { DDPStreamer } from './DDPStreamer';
+
+api.registerService(new DDPStreamer());

ee/apps/ddp-streamer/src/streams.ts

@@ -0,0 +1,23 @@
+import type { ISubscription, IRoom, IUser, ISetting } from '@rocket.chat/core-typings';
+
+import { NotificationsModule } from '../../../../apps/meteor/server/modules/notifications/notifications.module';
+import { Stream } from './Streamer';
+import { Collections, getConnection } from '../../../../apps/meteor/ee/server/services/mongo';
+import { RoomsRaw } from '../../../../apps/meteor/app/models/server/raw/Rooms';
+import { SubscriptionsRaw } from '../../../../apps/meteor/app/models/server/raw/Subscriptions';
+import { UsersRaw } from '../../../../apps/meteor/app/models/server/raw/Users';
+import { SettingsRaw } from '../../../../apps/meteor/app/models/server/raw/Settings';
+
+export const notifications = new NotificationsModule(Stream);
+
+getConnection().then((db) => {
+	const Users = new UsersRaw(db.collection<IUser>(Collections.User));
+	notifications.configure({
+		Rooms: new RoomsRaw(db.collection<IRoom>(Collections.Rooms)),
+		Subscriptions: new SubscriptionsRaw(db.collection<ISubscription>(Collections.Subscriptions), {
+			Users,
+		}),
+		Users,
+		Settings: new SettingsRaw(db.collection<ISetting>(Collections.Settings)),
+	});
+});

ee/apps/ddp-streamer/tsconfig.json

@@ -0,0 +1,29 @@
+{
+	"extends": "../../../tsconfig.base.json",
+	"compilerOptions": {
+		"target": "es2018",
+		"lib": ["esnext", "dom"],
+		"allowJs": true,
+		"checkJs": false,
+		"incremental": true,
+
+		/* Strict Type-Checking Options */
+		"noImplicitAny": true,
+		"strictNullChecks": true,
+		"strictPropertyInitialization": false,
+
+		/* Additional Checks */
+		"noUnusedLocals": true,
+		"noUnusedParameters": true,
+		"noImplicitReturns": false,
+		"noFallthroughCasesInSwitch": false,
+
+		/* Module Resolution Options */
+		"outDir": "./dist",
+		"importsNotUsedAsValues": "preserve",
+		"declaration": false,
+		"declarationMap": false
+	},
+	"include": ["./src/**/*", "../../../apps/meteor/definition", "./definition"],
+	"exclude": ["./dist", "./ecosystem.config.js"]
+}

package.json

@@ -21,6 +21,8 @@
 	"workspaces": [
 		"apps/*",
 		"packages/*",
+		"ee/apps/*",
+		"ee/packages/*",
 		"apps/meteor/ee/server/services"
 	],
 	"repository": {