Fix e2e error flakiness (#1803)

* Isolate test

* filter out as user.

* skip

* Add some message buffering

* Kill tests in a specific order.

* Add some retries for homerunner.

* remove describe annotations

* don't run docs on all pushes

* lint

* oops

* changelog

* Add a comment to describe what this does

* Add a github action reporter.

.github/workflows/docs-latest.yml

@@ -3,6 +3,7 @@ name: Build docs
 on:
   pull_request:
   push:
+    branches: ["develop", "release-*"]
 
 jobs:
   build:

changelog.d/1803.misc

@@ -0,0 +1 @@
+Improve e2e test reliability.

spec/e2e/jest.config.js

@@ -2,6 +2,7 @@
 module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
+  reporters: [['github-actions', {silent: false}], 'summary'],
   transformIgnorePatterns: ['<rootDir>/node_modules/'],
   testTimeout: 60000,
   transform: {

spec/e2e/quit.spec.ts

@@ -28,7 +28,7 @@ describe('Ensure quit messsage is sent', () => {
         const cRoomId = await testEnv.joinChannelHelper(alice, adminRoom, channel);
 
         // Ensure we join the IRC side
-        await alice.sendText(cRoomId, `Hello world!`);
+        alice.sendText(cRoomId, `Hello world!`);
         await bob.waitForEvent('message', 10000);
 
         const quitEvent = bob.waitForEvent('quit', 10000);
@@ -39,5 +39,7 @@ describe('Ensure quit messsage is sent', () => {
         expect(nick).toEqual(`M-${opts.matrixLocalparts?.[0]}`);
         expect(message).toEqual('Quit: Reconnecting');
         expect(channels).toContain(channel);
+        alice.sendText(cRoomId, `I'm back baby!`);
+        await bob.waitForEvent('message', 10000);
     });
 });

spec/e2e/replies.spec.ts

@@ -56,6 +56,7 @@ describe('Reply handling', () => {
         expect(ircMessage[2]).toContain("Short reply");
         expect(ircMessage[2]).not.toContain("Original message");
 
+        // Delay for the duration it takes to get a long reply.
         await new Promise(r => setTimeout(r, 1000));
 
         bridgedMessage = bob.waitForEvent('message', 10000);

spec/util/e2e-test.ts

@@ -366,14 +366,14 @@ export class IrcBridgeE2ETest {
     }
 
     public async tearDown(): Promise<void> {
-        await Promise.allSettled([
-            this.ircBridge?.kill(),
-            this.ircTest.tearDown(),
-            this.homeserver?.users.map(c => c.client.stop()),
-            this.homeserver && destroyHS(this.homeserver.id),
-            this.dropDatabase(),
-        ]);
+        // This is specifically ordered this way so that
+        // it's closed in dependency order.
+        await this.ircBridge?.kill();
+        await this.ircTest.tearDown();
+        await this.homeserver?.users.map(c => c.client.stop());
+        await (this.homeserver && destroyHS(this.homeserver.id));
         await this.pool?.close();
+        await this.dropDatabase();
         if (this.traceLog) {
             this.traceLog.close();
         }

spec/util/homerunner.ts

@@ -3,6 +3,7 @@ import { createHash, createHmac } from "crypto";
 import { Homerunner } from "homerunner-client";
 import { default as crossFetch } from 'cross-fetch';
 import { E2ETestMatrixClient } from "./e2e-test";
+import { delay } from "../../src/promiseutil";
 
 const HOMERUNNER_IMAGE = process.env.HOMERUNNER_IMAGE || 'complement-synapse';
 export const DEFAULT_REGISTRATION_SHARED_SECRET = (
@@ -69,28 +70,41 @@ export async function createHS(localparts: string[] = [], workerId: number): Pro
         SenderLocalpart: AppserviceConfig.senderLocalpart,
         RateLimited: false,
     };
-
-    const blueprintResponse = await homerunner.create({
-        base_image_uri: HOMERUNNER_IMAGE,
-        blueprint: {
-            Name: blueprint,
-            Homeservers: [{
-                Name: AppserviceConfig.id,
-                ApplicationServices: [{
-                    ...asRegistration,
-                    URL: `http://host.docker.internal:${AppserviceConfig.port}`,
-                }],
-                Users: localparts.map(localpart => ({Localpart: localpart, DisplayName: localpart})),
-            }],
+    let blueprintResponse: Awaited<ReturnType<Homerunner.Client["create"]>>|undefined;
+    let retries = 0;
+    do {
+        try {
+            blueprintResponse = await homerunner.create({
+                base_image_uri: HOMERUNNER_IMAGE,
+                blueprint: {
+                    Name: blueprint,
+                    Homeservers: [{
+                        Name: AppserviceConfig.id,
+                        ApplicationServices: [{
+                            ...asRegistration,
+                            URL: `http://host.docker.internal:${AppserviceConfig.port}`,
+                        }],
+                        Users: localparts.map(localpart => ({Localpart: localpart, DisplayName: localpart})),
+                    }],
+                }
+            });
+        }
+        catch (ex) {
+            console.warn("Failed to create deployment", ex);
+            retries++;
+            if (retries >= 5) {
+                throw ex;
+            }
+            await delay(1000);
         }
-    });
+    } while (!blueprintResponse)
     const [homeserverName, homeserver] = Object.entries(blueprintResponse.homeservers)[0];
     const users = Object.entries(homeserver.AccessTokens).map(([userId, accessToken]) => ({
         userId: userId,
         accessToken,
         deviceId: homeserver.DeviceIDs[userId],
         client: new E2ETestMatrixClient(homeserver.BaseURL, accessToken),
-    }));
+    })).filter(u => u.userId !== `@${AppserviceConfig.senderLocalpart}:${homeserverName}`);
 
     // Start syncing proactively.
     await Promise.all(users.map(u => u.client.start()));