Merge pull request #1035 from neet/redis-docker-compose

Use docker-compose for E2E testing

.github/workflows/ci.yml

@@ -46,30 +46,6 @@ jobs:
   test-e2e:
     name: E2E Test
     runs-on: ubuntu-latest
-    env:
-      DB_USER: mastodon
-      DB_NAME: mastodon
-      DB_PASS: password
-      MASTODON_CONTAINER: mastodon
-
-    services:
-      db:
-        image: postgres
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_USER: ${{ env.DB_USER }}
-          POSTGRES_DB: ${{ env.DB_NAME }}
-          POSTGRES_PASSWORD: ${{ env.DB_PASS }}
-          POSTGRES_HOST_AUTH_METHOD: trust
-        options: >-
-          --health-cmd "pg_isready -U postgres"
-      redis:
-        image: redis
-        ports:
-          - 6379:6379
-        options: >-
-          --health-cmd "redis-cli ping"
 
     steps:
       - uses: actions/checkout@v3
@@ -83,33 +59,24 @@ jobs:
           cache: yarn
 
       - name: Setup Mastodon
-        run: >-
-          docker run
-          -i
-          --rm
-          --env-file ./.github/.env.test
-          docker.io/neetshin/mastodon-dev:latest
-          bash -c "RAILS_ENV=development bundle exec rails db:setup"
-
-      - name: Run Mastodon
-        run: >-
-          docker run
-          -d
-          -p 3000:3000
-          -p 4000:4000
-          --name ${{ env.MASTODON_CONTAINER }} 
-          -e DEEPL_PLAN=${{ secrets.DEEPL_PLAN }}
-          -e DEEPL_API_KEY=${{ secrets.DEEPL_API_KEY }}
-          --env-file ./.github/.env.test
-          docker.io/neetshin/mastodon-dev:latest
-          bash -c "foreman start"
+        run: docker compose up -d
 
       - name: Install dependencies
         run: yarn install --frozen-lockfile
 
+      - name: Wait for http://localhost:3000
+        run: |
+          for i in {1..10}; do
+            curl -s -o /dev/null -w "%{http_code}" http://localhost:3000/api/v2/instance && break
+            sleep 5
+          done
+
       - name: Run tests
         run: yarn run test:e2e --max-workers=2
 
+      - name: Teardown Mastodon
+        run: docker compose down
+
       - name: Codecov
         uses: codecov/codecov-action@v3
         with:

cspell.json

@@ -1,10 +1,7 @@
 {
-  "version": "0.1",
+  "version": "0.2",
   "language": "en,en-GB",
-  "ignorePaths": [
-    "**/node_modules/**",
-    "**/dist/**"
-  ],
+  "ignorePaths": ["**/node_modules/**", "**/dist/**"],
   "words": [
     "AGPL",
     "asynckit",

docker-compose.yaml

@@ -0,0 +1,48 @@
+---
+version: "3"
+services:
+  db:
+    restart: always
+    image: postgres
+    ports:
+      - "5432:5432"
+    environment:
+      - POSTGRES_USER=mastodon
+      - POSTGRES_PASSWORD=mastodon
+      - POSTGRES_DB=mastodon
+      - POSTGRES_HOST_AUTH_METHOD=trust
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "postgres"]
+
+  redis:
+    restart: always
+    image: redis
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+
+  mastodon:
+    restart: always
+    image: neetshin/mastodon-dev:latest
+    ports:
+      - "3000:3000"
+      - "4000:4000"
+    depends_on:
+      - db
+      - redis
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "wget -q --spider --proxy=off localhost:3000/health || exit 1",
+        ]
+    env_file:
+      - .env.test
+    environment:
+      - RAILS_ENV=development
+    command: >
+      /bin/bash -c "
+        bundle exec rails db:setup &&
+        foreman start
+      "

package.json

@@ -45,7 +45,6 @@
     "@size-limit/preset-small-lib": "^10.0.1",
     "@types/jest": "^29.5.6",
     "@types/node": "^20.8.9",
-    "@types/proper-lockfile": "^4.1.3",
     "@types/ws": "^8.5.8",
     "@typescript-eslint/eslint-plugin": "^6.9.0",
     "@typescript-eslint/parser": "^6.9.0",
@@ -58,11 +57,11 @@
     "eslint-plugin-simple-import-sort": "^10.0.0",
     "eslint-plugin-unicorn": "^48.0.1",
     "get-port": "^5.1.1",
+    "ioredis": "^5.3.2",
     "iterator-helpers-polyfill": "^2.3.3",
     "jest": "^29.6.4",
     "npm-run-all": "^4.1.5",
     "prettier": "^3.0.3",
-    "proper-lockfile": "^4.1.2",
     "rollup": "^4.1.4",
     "rollup-plugin-auto-external": "^2.0.0",
     "rollup-plugin-dts": "^6.1.0",
@@ -73,7 +72,7 @@
     "tslib": "^2.6.2",
     "typedoc": "^0.25.2",
     "typescript": "^5.2.2",
-    "undici": "^5.27.0"
+    "undici": "^6.6.2"
   },
   "files": [
     "README.md",

src/adapters/action/dispatcher-ws.ts

@@ -26,7 +26,7 @@ export class WebSocketActionDispatcher implements ActionDispatcher {
     }
 
     if (action.type !== "subscribe") {
-      throw new MastoUnexpectedError("Unknown action type");
+      throw new MastoUnexpectedError(`Unknown action type ${action.type}`);
     }
 
     const data = action.data ?? {};

src/mastodon/streaming/client.ts

@@ -58,6 +58,7 @@ export interface Client {
   };
 
   close(): void;
+
   /** @internal */
   prepare(): Promise<void>;
 }

test-utils/jest-environment.ts

@@ -3,24 +3,34 @@ import { existsSync } from "node:fs";
 import fs from "node:fs/promises";
 import path from "node:path";
 
+import Redis from "ioredis";
 import NodeEnvironment from "jest-environment-node";
 
 import {
   createOAuthAPIClient,
   createRestAPIClient,
   type mastodon,
 } from "../src";
-import { TokenPoolFsImpl } from "./pools";
-import { TokenFactoryDocker } from "./pools/token-factory-docker";
-import { TokenRepositoryFs } from "./pools/token-repository-fs";
+import { TokenFactoryDocker, TokenPoolRedis } from "./pools";
 import { createTootctl } from "./tootctl";
 
 class CustomEnvironment extends NodeEnvironment {
+  redis!: Redis;
+
   override async setup(): Promise<void> {
     await super.setup();
-    const misc = await this.createGlobals();
-    this.global.__misc__ = misc;
-    this.global.Symbol = Symbol;
+    this.redis = new Redis();
+    this.global.__misc__ = await this.createGlobals();
+
+    /* eslint-disable @typescript-eslint/no-explicit-any */
+    (this.global.Symbol as any).dispose ??= Symbol("Symbol.dispose");
+    (this.global.Symbol as any).asyncDispose ??= Symbol("Symbol.asyncDispose");
+    /* eslint-enable @typescript-eslint/no-explicit-any */
+  }
+
+  override async teardown(): Promise<void> {
+    await super.teardown();
+    this.redis.disconnect();
   }
 
   private async createGlobals(): Promise<typeof globalThis.__misc__> {
@@ -33,15 +43,12 @@ class CustomEnvironment extends NodeEnvironment {
     }
 
     const adminToken = await this.readAdminToken(baseCacheDir);
-    const repository = new TokenRepositoryFs(
-      path.join(baseCacheDir, "tokens.json"),
-    );
     const container = process.env.MASTODON_CONTAINER ?? "mastodon";
-    const tootctl = createTootctl({ container });
+    const tootctl = createTootctl({ container, compose: true });
     const oauth = createOAuthAPIClient({ url });
     const app = await this.readApp(baseCacheDir);
     const factory = new TokenFactoryDocker(tootctl, oauth, app);
-    const tokenPool = new TokenPoolFsImpl(repository, factory);
+    const tokenPool = new TokenPoolRedis(this.redis, factory);
 
     return {
       url,

test-utils/jest-global-setup.ts

@@ -57,16 +57,6 @@ const readOrCreateAdminToken = async (
   return token;
 };
 
-const initTokens = async (baseCacheDir: string) => {
-  const tokensFilePath = path.join(baseCacheDir, "tokens.json");
-
-  if (existsSync(tokensFilePath)) {
-    return;
-  }
-
-  await fs.writeFile(path.join(baseCacheDir, "tokens.json"), "[]", "utf8");
-};
-
 export default async function main(): Promise<void> {
   const baseCacheDir = path.join(__dirname, "../node_modules/.cache/masto");
   if (!existsSync(baseCacheDir)) {
@@ -78,5 +68,4 @@ export default async function main(): Promise<void> {
 
   const app = await readOrCreateApp(baseCacheDir, masto);
   await readOrCreateAdminToken(baseCacheDir, oauth, app);
-  await initTokens(baseCacheDir);
 }

test-utils/pools/index.ts

@@ -1,4 +1,5 @@
 export * from "./session-pool";
 export * from "./token-pool";
-export * from "./token-pool-fs";
+export * from "./token-pool-impl";
+export * from "./token-factory-docker";
 export * from "./base-pool";

test-utils/pools/session-pool.ts

@@ -3,6 +3,10 @@ import { createSession, type Session } from "../session";
 import { type Pool } from "./base-pool";
 import { type TokenPool } from "./token-pool";
 
+export type AcquireOption = {
+  readonly waitForWs?: boolean;
+};
+
 export class SessionPoolImpl implements Pool<Session> {
   private readonly sessionToToken = new WeakMap<Session, mastodon.v1.Token>();
 
@@ -12,24 +16,34 @@ export class SessionPoolImpl implements Pool<Session> {
     private readonly instance: mastodon.v1.Instance,
   ) {}
 
-  acquire = async (): Promise<Session> => {
-    const token = await this.tokens.acquire();
-
+  acquire = async (options: AcquireOption = {}): Promise<Session> => {
     try {
-      const session = await createSession(
-        token,
-        this.url,
-        this.instance.urls.streamingApi,
-        () => this.release(session),
-      );
+      const token = await this.tokens.acquire();
 
-      // eslint-disable-next-line no-console
-      console.log(`Acquired session ${session.id} (${session.acct})`);
+      try {
+        const session = await createSession(
+          token,
+          this.url,
+          this.instance.urls.streamingApi,
+          () => this.release(session),
+        );
 
-      this.sessionToToken.set(session, token);
-      return session;
+        this.sessionToToken.set(session, token);
+
+        if (options.waitForWs) {
+          await session.ws.prepare();
+        }
+
+        return session;
+      } catch (error) {
+        await this.tokens.release(token);
+        // eslint-disable-next-line no-console
+        console.error(error);
+        throw error;
+      }
     } catch (error) {
-      await this.tokens.release(token);
+      // eslint-disable-next-line no-console
+      console.error(error);
       throw error;
     }
   };
@@ -45,9 +59,6 @@ export class SessionPoolImpl implements Pool<Session> {
       return;
     }
 
-    // eslint-disable-next-line no-console
-    console.log(`Released session ${session.id} (${session.acct})`);
-
     await this.tokens.release(token);
     this.sessionToToken.delete(session);
   };