Fix AI bot messages rendered as user messages on staging

.github/workflows/ci.yaml

@@ -535,6 +535,7 @@ jobs:
             "server-endpoints/search-test.ts",
             "server-endpoints/info-test.ts",
             "server-endpoints/search-prerendered-test.ts",
+            "server-config-test.ts",
             "server-endpoints/stripe-session-test.ts",
             "server-endpoints/stripe-webhook-test.ts",
             "server-endpoints/user-and-catalog-test.ts",

packages/matrix/docker/synapse/index.ts

@@ -1,6 +1,7 @@
 import * as path from 'path';
 import * as os from 'os';
 import * as crypto from 'crypto';
+import * as net from 'net';
 import * as fse from 'fs-extra';
 import { request } from '@playwright/test';
 import {
@@ -40,14 +41,58 @@ export interface SynapseInstance extends SynapseConfig {
 }
 
 const synapses = new Map<string, SynapseInstance>();
+const dynamicHostPortStartAttempts = 5;
+
+function findAvailablePort(preferred?: number): Promise<number> {
+  return new Promise((resolve, reject) => {
+    let server = net.createServer();
+
+    server.on('error', (error: NodeJS.ErrnoException) => {
+      server.close();
+      if (preferred != null && error.code === 'EADDRINUSE') {
+        findAvailablePort(undefined).then(resolve, reject);
+        return;
+      }
+      reject(error);
+    });
+
+    server.listen(preferred ?? 0, '127.0.0.1', () => {
+      let address = server.address();
+      if (!address || typeof address === 'string') {
+        server.close(() =>
+          reject(new Error('Could not determine available port')),
+        );
+        return;
+      }
+      let { port } = address;
+      server.close((error) => {
+        if (error) {
+          reject(error);
+          return;
+        }
+        resolve(port);
+      });
+    });
+  });
+}
 
 function randB64Bytes(numBytes: number): string {
   return crypto.randomBytes(numBytes).toString('base64').replace(/=*$/, '');
 }
 
+function isPortBindError(error: unknown): boolean {
+  let message = error instanceof Error ? error.message : String(error);
+  return /address already in use|port is already allocated/i.test(message);
+}
+
 export async function cfgDirFromTemplate(
   template: string,
   dataDir?: string,
+  options?: {
+    publicBaseUrl?: string;
+    host?: string;
+    port?: number;
+  },
 ): Promise<SynapseConfig> {
   const templateDir = path.join(__dirname, template);
 
@@ -69,7 +114,9 @@ export async function cfgDirFromTemplate(
   const macaroonSecret = randB64Bytes(16);
   const formSecret = randB64Bytes(16);
 
-  const baseUrl = `http://${SYNAPSE_IP_ADDRESS}:${SYNAPSE_PORT}`;
+  const host = options?.host ?? SYNAPSE_IP_ADDRESS;
+  const port = options?.port ?? SYNAPSE_PORT;
+  const baseUrl = options?.publicBaseUrl ?? `http://${host}:${port}`;
 
   // now copy homeserver.yaml, applying substitutions
   console.log(`Gen ${path.join(templateDir, 'homeserver.yaml')}`);
@@ -95,8 +142,8 @@ export async function cfgDirFromTemplate(
   );
 
   return {
-    port: SYNAPSE_PORT,
-    host: SYNAPSE_IP_ADDRESS,
+    port,
+    host,
     baseUrl,
     configDir,
     registrationSecret,
@@ -136,51 +183,80 @@ export async function synapseStart(
     }
     await Promise.allSettled(stopPromises);
   }
-  const synCfg = await cfgDirFromTemplate(
-    opts?.template ?? 'test',
-    opts?.dataDir,
-  );
-  let containerName =
-    opts?.containerName ||
-    (isEnvironmentMode()
-      ? getSynapseContainerName()
-      : path.basename(synCfg.configDir));
-  console.log(
-    `Starting synapse with config dir ${synCfg.configDir} in container ${containerName}...`,
+  let useDynamicHostPort = Boolean(
+    isEnvironmentMode() || opts?.dynamicHostPort,
   );
   await dockerCreateNetwork({ networkName: 'boxel' });
 
-  let dockerParams: string[] = [
-    '--rm',
-    '-v',
-    `${synCfg.configDir}:/data`,
-    '-v',
-    `${path.join(__dirname, 'templates')}:/custom/templates/`,
-  ];
-  if (isEnvironmentMode() || opts?.dynamicHostPort) {
-    // Dynamic host port, with fixed container IP only when not running in branch mode
-    if (!isEnvironmentMode()) {
-      dockerParams.push(`--ip=${synCfg.host}`);
-    }
-    dockerParams.push('-p', '0:8008/tcp', '--network=boxel');
-  } else {
-    dockerParams.push(
-      `--ip=${synCfg.host}`,
-      '-p',
-      `${synCfg.port}:8008/tcp`,
-      '--network=boxel',
+  let hostPort = SYNAPSE_PORT;
+  let synCfg!: SynapseConfig;
+  let containerName!: string;
+  let synapseId!: string;
+  let attempts = useDynamicHostPort ? dynamicHostPortStartAttempts : 1;
+
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    hostPort = useDynamicHostPort ? await findAvailablePort() : SYNAPSE_PORT;
+    synCfg = await cfgDirFromTemplate(opts?.template ?? 'test', opts?.dataDir, {
+      host: useDynamicHostPort ? '127.0.0.1' : SYNAPSE_IP_ADDRESS,
+      port: hostPort,
+      publicBaseUrl: `http://localhost:${hostPort}`,
+    });
+    containerName =
+      opts?.containerName ||
+      (isEnvironmentMode()
+        ? getSynapseContainerName()
+        : path.basename(synCfg.configDir));
+    console.log(
+      `Starting synapse with config dir ${synCfg.configDir} in container ${containerName}...`,
     );
-  }
 
-  const synapseId = await dockerRun({
-    image: 'matrixdotorg/synapse:v1.126.0',
-    containerName,
-    dockerParams,
-    applicationParams: ['run'],
-    runAsUser: true,
-  });
+    let dockerParams: string[] = [
+      '--rm',
+      '-v',
+      `${synCfg.configDir}:/data`,
+      '-v',
+      `${path.join(__dirname, 'templates')}:/custom/templates/`,
+    ];
+    if (useDynamicHostPort) {
+      // In dynamic-host-port mode multiple harnesses may run concurrently, so
+      // we must not claim the shared fixed Synapse container IP.
+      dockerParams.push('-p', `${hostPort}:8008/tcp`, '--network=boxel');
+    } else {
+      dockerParams.push(
+        `--ip=${synCfg.host}`,
+        '-p',
+        `${synCfg.port}:8008/tcp`,
+        '--network=boxel',
+      );
+    }
 
-  console.log(`Started synapse with id ${synapseId} on port ${synCfg.port}`);
+    try {
+      synapseId = await dockerRun({
+        image: 'matrixdotorg/synapse:v1.126.0',
+        containerName,
+        dockerParams,
+        applicationParams: ['run'],
+        runAsUser: true,
+      });
+      break;
+    } catch (error) {
+      if (
+        !useDynamicHostPort ||
+        !isPortBindError(error) ||
+        attempt === attempts
+      ) {
+        throw error;
+      }
+      console.warn(
+        `Synapse host port ${hostPort} was claimed before Docker bound it; retrying (${attempt}/${attempts})...`,
+      );
+      if (!opts?.dataDir) {
+        await fse.remove(synCfg.configDir);
+      }
+    }
+  }
+
+  console.log(`Started synapse with id ${synapseId} on port ${hostPort}`);
 
   // Await Synapse healthcheck
   await dockerExec({
@@ -199,9 +275,13 @@ export async function synapseStart(
     ],
   });
 
-  let hostPort = synCfg.port;
-  if (isEnvironmentMode() || opts?.dynamicHostPort) {
-    hostPort = await resolveHostPort(synapseId);
+  if (useDynamicHostPort) {
+    let resolvedPort = await resolveHostPort(synapseId);
+    if (resolvedPort !== hostPort) {
+      throw new Error(
+        `Synapse started on unexpected host port ${resolvedPort}; expected ${hostPort}`,
+      );
+    }
     console.log(`Synapse dynamic host port: ${hostPort}`);
   }
 

packages/realm-server/scripts/start-production.sh

@@ -25,6 +25,7 @@ EXTERNAL_CATALOG_REALM_URL="${RESOLVED_EXTERNAL_CATALOG_REALM_URL:-$DEFAULT_EXTE
 NODE_NO_WARNINGS=1 \
   LOW_CREDIT_THRESHOLD=2000 \
   MATRIX_URL=https://matrix.boxel.ai \
+  MATRIX_SERVER_NAME=boxel.ai \
   BOXEL_HOST_URL=https://app.boxel.ai \
   REALM_SERVER_MATRIX_USERNAME=realm_server \
   PUBLISHED_REALM_BOXEL_SPACE_DOMAIN='boxel.space' \

packages/realm-server/scripts/start-staging.sh

@@ -25,6 +25,7 @@ EXTERNAL_CATALOG_REALM_URL="${RESOLVED_EXTERNAL_CATALOG_REALM_URL:-$DEFAULT_EXTE
 NODE_NO_WARNINGS=1 \
   LOW_CREDIT_THRESHOLD=2000 \
   MATRIX_URL=https://matrix-staging.stack.cards \
+  MATRIX_SERVER_NAME=stack.cards \
   BOXEL_HOST_URL=https://realms-staging.stack.cards \
   REALM_SERVER_MATRIX_USERNAME=realm_server \
   PUBLISHED_REALM_BOXEL_SPACE_DOMAIN='staging.boxel.dev' \

packages/realm-server/server.ts

@@ -707,7 +707,9 @@ export class RealmServer {
           hostsOwnAssets: false,
           assetsURL: this.assetsURL.href,
           matrixURL: this.matrixClient.matrixURL.href.replace(/\/$/, ''),
-          matrixServerName: this.matrixClient.matrixURL.hostname,
+          matrixServerName:
+            process.env.MATRIX_SERVER_NAME ||
+            this.matrixClient.matrixURL.hostname,
           realmServerURL: this.serverURL.href,
           resolvedBaseRealmURL: rewriteRealmURL(config.resolvedBaseRealmURL),
           resolvedCatalogRealmURL: rewriteRealmURL(

packages/realm-server/tests/index.ts

@@ -158,6 +158,7 @@ import './server-endpoints/queue-status-test';
 import './server-endpoints/realm-lifecycle-test';
 import './server-endpoints/search-test';
 import './server-endpoints/search-prerendered-test';
+import './server-config-test';
 import './server-endpoints/info-test';
 import './server-endpoints/stripe-session-test';
 import './server-endpoints/stripe-webhook-test';

packages/realm-server/tests/server-config-test.ts

@@ -0,0 +1,65 @@
+import { module, test } from 'qunit';
+import { basename } from 'path';
+import { dirSync } from 'tmp';
+
+import { RealmServer } from '../server';
+
+module(basename(__filename), function () {
+  test('prefers MATRIX_SERVER_NAME over matrix URL hostname in host config', async function (assert) {
+    let originalMatrixServerName = process.env.MATRIX_SERVER_NAME;
+    let tempDir = dirSync({ unsafeCleanup: true });
+
+    process.env.MATRIX_SERVER_NAME = 'stack.cards';
+
+    try {
+      let server = new RealmServer({
+        serverURL: new URL('http://127.0.0.1:4448'),
+        realms: [],
+        virtualNetwork: {} as any,
+        matrixClient: {
+          matrixURL: new URL('http://localhost:8008/'),
+        } as any,
+        realmServerSecretSeed: 'test-realm-server-secret',
+        realmSecretSeed: 'test-realm-secret',
+        grafanaSecret: 'test-grafana-secret',
+        realmsRootPath: tempDir.name,
+        dbAdapter: {} as any,
+        queue: {} as any,
+        definitionLookup: {} as any,
+        assetsURL: new URL('http://example.com/notional-assets-host/'),
+        matrixRegistrationSecret: 'test-matrix-registration-secret',
+        getIndexHTML: async () =>
+          `<html><head><meta name="@cardstack/host/config/environment" content="${encodeURIComponent(
+            JSON.stringify({
+              matrixURL: 'http://localhost:8008',
+              matrixServerName: 'localhost',
+              realmServerURL: 'http://localhost:4201/',
+              publishedRealmBoxelSpaceDomain: 'localhost:4201',
+              publishedRealmBoxelSiteDomain: 'localhost:4201',
+            }),
+          )}"></head><body></body></html>`,
+      });
+
+      let html = await (server as any).retrieveIndexHTML();
+      let match = html.match(
+        /<meta name="@cardstack\/host\/config\/environment" content="([^"]+)">/,
+      );
+
+      assert.ok(match, 'host config environment meta tag is present');
+
+      let config = JSON.parse(decodeURIComponent(match![1]));
+      assert.strictEqual(
+        config.matrixServerName,
+        'stack.cards',
+        'uses MATRIX_SERVER_NAME override in host config',
+      );
+    } finally {
+      if (originalMatrixServerName == null) {
+        delete process.env.MATRIX_SERVER_NAME;
+      } else {
+        process.env.MATRIX_SERVER_NAME = originalMatrixServerName;
+      }
+      tempDir.removeCallback();
+    }
+  });
+});

packages/software-factory/src/harness/isolated-realm-stack.ts

@@ -341,6 +341,7 @@ export async function startIsolatedRealmStack({
     GRAFANA_SECRET,
     HOST_URL: context.hostURL,
     MATRIX_URL: context.matrixURL,
+    MATRIX_SERVER_NAME: new URL(context.matrixURL).hostname,
     MATRIX_REGISTRATION_SHARED_SECRET: context.matrixRegistrationSecret,
     REALM_SERVER_MATRIX_USERNAME: DEFAULT_MATRIX_SERVER_USERNAME,
     REALM_SERVER_FULL_INDEX_ON_STARTUP: String(fullIndexOnStartup),