chore(test): Enable concurrent test

fix race condition

jest.config.cjs

@@ -9,7 +9,7 @@ module.exports = {
     },
     {
       displayName: 'e2e',
-      testEnvironment: 'node',
+      testEnvironment: './test-utils/jest-environment.ts',
       testMatch: ['<rootDir>/tests/**/*.spec.ts'],
       transform: { '^.+\\.tsx?$': 'ts-jest' },
       globalSetup: '<rootDir>/test-utils/jest-global-setup.ts',

package.json

@@ -19,13 +19,14 @@
   "scripts": {
     "test": "npm-run-all test:*",
     "test:unit": "jest --coverage --config=jest.config.cjs --selectProjects unit",
-    "test:e2e": "jest --coverage --runInBand --detectOpenHandles --config=jest.config.cjs --selectProjects e2e",
+    "test:e2e": "jest --coverage --config=jest.config.cjs --selectProjects e2e",
     "lint": "npm-run-all lint:*",
     "lint:eslint": "eslint -c .eslintrc.json ./src/**/*.ts --cache",
     "lint:spellcheck": "cspell '{src,examples}/**/*.{ts,tsx,js,json,md}'",
     "build": "rollup -c rollup.config.cjs",
     "prepublishOnly": "yarn run build",
-    "docs:build": "typedoc ./src/index.ts && touch ./docs/.nojekyll"
+    "docs:build": "typedoc ./src/index.ts && touch ./docs/.nojekyll",
+    "purge:cache": "rm -rf ./node_modules/.cache/masto"
   },
   "dependencies": {
     "@mastojs/ponyfills": "^1.0.4",
@@ -44,6 +45,7 @@
     "@types/jest": "^29.5.1",
     "@types/node": "^18.13.0",
     "@types/parse-link-header": "^2.0.1",
+    "@types/proper-lockfile": "^4.1.2",
     "@types/ws": "^8.5.4",
     "@typescript-eslint/eslint-plugin": "^5.59.2",
     "@typescript-eslint/parser": "^5.59.2",
@@ -55,11 +57,11 @@
     "eslint-plugin-prettier": "^4.0.0",
     "eslint-plugin-simple-import-sort": "^10.0.0",
     "eslint-plugin-unicorn": "^45.0.2",
-    "generic-pool": "^3.9.0",
     "iterator-helpers-polyfill": "^2.2.8",
     "jest": "^29.5.0",
     "npm-run-all": "^4.1.5",
     "prettier": "^2.8.4",
+    "proper-lockfile": "^4.1.2",
     "rollup": "^3.15.0",
     "rollup-plugin-auto-external": "^2.0.0",
     "rollup-plugin-dts": "^5.3.0",

src/utils/exponential-backoff.ts

@@ -4,10 +4,13 @@ import { delay } from './delay';
 export class ExponentialBackoff {
   private _attempts = 0;
 
-  constructor(private readonly baseSeconds: number) {}
+  constructor(
+    private readonly base: number = 2,
+    private readonly factor = 1000,
+  ) {}
 
   get timeout(): number {
-    return this.baseSeconds ** this._attempts * 1000;
+    return this.factor * this.base ** this._attempts;
   }
 
   get attempts(): number {

test-utils/global.d.ts

@@ -1,16 +1,15 @@
 /* eslint-disable no-var */
 import type { mastodon } from '../src';
-import type { SessionPoolImpl, TokenPool } from './pools';
+import type { SessionPoolImpl } from './pools';
 
 declare global {
   var admin: mastodon.RestClient;
   var sessions: SessionPoolImpl;
 
-  /** Should only be used inside /test-utils */
   var __misc__: {
     url: string;
+    tokens: TokenPool;
     instance: mastodon.v1.Instance;
     adminToken: mastodon.v1.Token;
-    tokens: TokenPool;
   };
 }

test-utils/jest-environment.ts

@@ -0,0 +1,13 @@
+import NodeEnvironment from 'jest-environment-node';
+
+import { createMisc } from './misc';
+
+class CustomEnvironment extends NodeEnvironment {
+  override async setup(): Promise<void> {
+    await super.setup();
+    const misc = await createMisc();
+    this.global.__misc__ = misc;
+  }
+}
+
+export default CustomEnvironment;

test-utils/jest-global-setup.ts

@@ -1,25 +1,54 @@
+/* eslint-disable no-console */
+/* eslint-disable unicorn/prefer-module */
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
-import './jest-polyfills';
+import crypto from 'node:crypto';
+import { existsSync } from 'node:fs';
+import fs from 'node:fs/promises';
+import path from 'node:path';
 
+import type { Config } from '@jest/types';
+
+import type { mastodon } from '../src';
 import { createOAuthClient, createRestClient } from '../src';
-import { TokenPoolImpl } from './pools';
+import type { TokenRepository } from './pools/token-repository';
+import { TokenRepositoryFs } from './pools/token-repository-fs';
+import type { Tootctl } from './tootctl';
+import { createTootctl } from './tootctl';
+
+const readOrCreateApp = async (
+  baseCacheDir: string,
+  masto: mastodon.RestClient,
+): Promise<mastodon.v1.Client> => {
+  const appFilePath = path.join(baseCacheDir, 'app.json');
 
-export default async (): Promise<void> => {
-  const url = 'http://localhost:3000';
-  const instance = await createRestClient({ url }).v1.instance.fetch();
-  const masto = createRestClient({ url });
-  const oauth = createOAuthClient({ url });
+  if (existsSync(appFilePath)) {
+    const json = await fs.readFile(appFilePath, 'utf8');
+    return JSON.parse(json);
+  }
 
   const app = await masto.v1.apps.create({
     clientName: 'Masto.js',
     redirectUris: 'urn:ietf:wg:oauth:2.0:oob',
     scopes: 'read write follow push admin:read admin:write',
   });
 
-  const container = process.env.MASTODON_CONTAINER ?? 'mastodon';
-  const tokenPool = new TokenPoolImpl(container, oauth, app);
+  fs.writeFile(appFilePath, JSON.stringify(app, undefined, 2));
+  return app;
+};
 
-  const adminToken = await oauth.token.create({
+const readOrCreateAdminToken = async (
+  baseCacheDir: string,
+  oauth: mastodon.OAuthClient,
+  app: mastodon.v1.Client,
+): Promise<mastodon.v1.Token> => {
+  const tokenFilePath = path.join(baseCacheDir, 'admin_token.json');
+
+  if (existsSync(tokenFilePath)) {
+    const json = await fs.readFile(tokenFilePath, 'utf8');
+    return JSON.parse(json);
+  }
+
+  const token = await oauth.token.create({
     grantType: 'password',
     clientId: app.clientId!,
     clientSecret: app.clientSecret!,
@@ -28,10 +57,81 @@ export default async (): Promise<void> => {
     scope: 'read write follow push admin:read admin:write',
   });
 
-  globalThis.__misc__ = {
-    url,
-    instance,
-    tokens: tokenPool,
-    adminToken,
-  };
+  fs.writeFile(tokenFilePath, JSON.stringify(token, undefined, 2));
+  return token;
+};
+
+const createToken = async (
+  tootctl: Tootctl,
+  app: mastodon.v1.Client,
+  oauth: mastodon.OAuthClient,
+) => {
+  const username = crypto.randomBytes(8).toString('hex');
+  const email = crypto.randomBytes(8).toString('hex') + '@example.com';
+
+  const { password } = await tootctl.accounts.create(username, {
+    email,
+    confirmed: true,
+  });
+
+  if (app.clientId == undefined || app.clientSecret == undefined) {
+    throw new Error('App not created');
+  }
+
+  const token = await oauth.token.create({
+    grantType: 'password',
+    clientId: app.clientId,
+    clientSecret: app.clientSecret,
+    username: email,
+    password,
+    scope: 'read write follow push admin:read admin:write',
+  });
+
+  return token;
 };
+
+const replenishTokens = async (
+  maxTokens: number,
+  tootctl: Tootctl,
+  app: mastodon.v1.Client,
+  oauth: mastodon.OAuthClient,
+  repository: TokenRepository,
+) => {
+  const tokens = await repository.getAll();
+
+  if (tokens.length >= maxTokens) {
+    return;
+  }
+
+  const maxConcurrency = 5;
+
+  const tasks = Array.from({ length: maxTokens - tokens.length }, (_, i) => {
+    return async () => {
+      console.log(`Replenishing tokens... ${i + 1}/${maxTokens}`);
+      const token = await createToken(tootctl, app, oauth);
+      await repository.add({ token, inUse: false });
+    };
+  });
+
+  while (tasks.length > 0) {
+    await Promise.all(tasks.splice(0, maxConcurrency).map((f) => f()));
+  }
+};
+
+export default async function main(config: Config.GlobalConfig): Promise<void> {
+  const baseCacheDir = path.join(__dirname, '../node_modules/.cache/masto');
+  if (!existsSync(baseCacheDir)) {
+    await fs.mkdir(baseCacheDir, { recursive: true });
+  }
+
+  const masto = createRestClient({ url: 'http://localhost:3000' });
+  const oauth = createOAuthClient({ url: 'http://localhost:3000' });
+  const container = process.env.MASTODON_CONTAINER ?? 'mastodon';
+  const repository = new TokenRepositoryFs(
+    path.join(baseCacheDir, 'tokens.json'),
+  );
+  const tootctl = createTootctl({ container });
+  const app = await readOrCreateApp(baseCacheDir, masto);
+  await replenishTokens(config.maxWorkers, tootctl, app, oauth, repository);
+  await readOrCreateAdminToken(baseCacheDir, oauth, app);
+}

test-utils/jest-setup-after-env.ts

@@ -7,8 +7,12 @@ import { SessionPoolImpl } from './pools';
 jest.setTimeout(1000 * 60);
 
 globalThis.admin = createRestClient({
-  url: __misc__.url,
-  accessToken: __misc__.adminToken.accessToken,
+  url: globalThis.__misc__.url,
+  accessToken: globalThis.__misc__.adminToken.accessToken,
 });
 
-globalThis.sessions = new SessionPoolImpl();
+globalThis.sessions = new SessionPoolImpl(
+  globalThis.__misc__.tokens,
+  globalThis.__misc__.url,
+  globalThis.__misc__.instance,
+);

test-utils/misc.ts

@@ -0,0 +1,55 @@
+/* eslint-disable unicorn/prefer-module */
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+import './jest-polyfills';
+
+import { existsSync } from 'node:fs';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import type { mastodon } from '../src';
+import { createRestClient } from '../src';
+import { TokenPoolFsImpl } from './pools';
+import { TokenRepositoryFs } from './pools/token-repository-fs';
+
+const readAdminToken = async (
+  baseCacheDir: string,
+): Promise<mastodon.v1.Token> => {
+  const tokenFilePath = path.join(baseCacheDir, 'admin_token.json');
+
+  if (!existsSync(tokenFilePath)) {
+    throw new Error('Admin token does not exist');
+  }
+
+  const json = await fs.readFile(tokenFilePath, 'utf8');
+  return JSON.parse(json);
+};
+
+type Misc = {
+  readonly url: string;
+  readonly instance: mastodon.v1.Instance;
+  readonly tokens: TokenPoolFsImpl;
+  readonly adminToken: mastodon.v1.Token;
+};
+
+export const createMisc = async (): Promise<Misc> => {
+  const url = 'http://localhost:3000';
+  const instance = await createRestClient({ url }).v1.instance.fetch();
+
+  const baseCacheDir = path.join(__dirname, '../node_modules/.cache/masto');
+  if (!existsSync(baseCacheDir)) {
+    throw new Error('Cache directory does not exist');
+  }
+
+  const adminToken = await readAdminToken(baseCacheDir);
+  const repository = new TokenRepositoryFs(
+    path.join(baseCacheDir, 'tokens.json'),
+  );
+  const tokenPool = new TokenPoolFsImpl(repository);
+
+  return {
+    url,
+    instance,
+    tokens: tokenPool,
+    adminToken,
+  };
+};