feat: Implement multi token support for client tokens

This adds support for multi project tokens to be created. Backward compatibility is handled at 3 different layers here:

- The API is made backwards compatible though a permissive data type that accepts either a project?: string or projects?: string[] property, validation is done through JOI here, which ensures that projects and project are not set together. In the case of neither, this defaults to the previous default of ALL_PROJECTS
- The service layer method to handle adding tokens has been made tolerant to either of the above case and has been deprecated, a new method supporting only the new structure of using projects has been added
- Existing compatibility for consumers of Unleash as a library should not be affected either, the ApiUser constructor is now tolerant to the the first input and will internally map to the new cleaned structure

src/lib/create-config.ts

@@ -21,7 +21,11 @@ import { defaultCustomAuthDenyAll } from './default-custom-auth-deny-all';
 import { formatBaseUri } from './util/format-base-uri';
 import { minutesToMilliseconds, secondsToMilliseconds } from 'date-fns';
 import EventEmitter from 'events';
-import { ApiTokenType, validateApiToken } from './types/models/api-token';
+import {
+    ApiTokenType,
+    mapLegacyToken,
+    validateApiToken,
+} from './types/models/api-token';
 
 const safeToUpper = (s: string) => (s ? s.toUpperCase() : s);
 
@@ -198,7 +202,7 @@ const loadTokensFromString = (tokenString: String, tokenType: ApiTokenType) => {
             type: tokenType,
             username: 'admin',
         };
-        validateApiToken(token);
+        validateApiToken(mapLegacyToken(token));
         return token;
     });
     return tokens;

src/lib/db/api-token-store.ts

@@ -9,13 +9,16 @@ import {
     ApiTokenType,
     IApiToken,
     IApiTokenCreate,
+    isAllProjects,
 } from '../types/models/api-token';
+import { ALL_PROJECTS } from '../../lib/services/access-service';
 
 const TABLE = 'api_tokens';
+const API_LINK_TABLE = 'api_token_project';
 
 const ALL = '*';
 
-interface ITokenTable {
+interface ITokenInsert {
     id: number;
     secret: string;
     username: string;
@@ -24,28 +27,50 @@ interface ITokenTable {
     created_at: Date;
     seen_at?: Date;
     environment: string;
+}
+
+interface ITokenRow extends ITokenInsert {
     project: string;
 }
 
+const tokenRowReducer = (acc, tokenRow) => {
+    const { project, ...token } = tokenRow;
+    if (!acc[tokenRow.secret]) {
+        acc[tokenRow.secret] = {
+            secret: token.secret,
+            username: token.username,
+            type: token.type,
+            project: ALL,
+            projects: [ALL],
+            environment: token.environment ? token.environment : ALL,
+            expiresAt: token.expires_at,
+            createdAt: token.created_at,
+        };
+    }
+    const currentToken = acc[tokenRow.secret];
+    if (tokenRow.project) {
+        if (isAllProjects(currentToken.projects)) {
+            currentToken.projects = [];
+        }
+        currentToken.projects.push(tokenRow.project);
+        currentToken.project = currentToken.projects.join(',');
+    }
+    return acc;
+};
+
 const toRow = (newToken: IApiTokenCreate) => ({
     username: newToken.username,
     secret: newToken.secret,
     type: newToken.type,
-    project: newToken.project === ALL ? undefined : newToken.project,
     environment:
         newToken.environment === ALL ? undefined : newToken.environment,
     expires_at: newToken.expiresAt,
 });
 
-const toToken = (row: ITokenTable): IApiToken => ({
-    secret: row.secret,
-    username: row.username,
-    type: row.type,
-    environment: row.environment ? row.environment : ALL,
-    project: row.project ? row.project : ALL,
-    expiresAt: row.expires_at,
-    createdAt: row.created_at,
-});
+const toTokens = (rows: any[]): IApiToken[] => {
+    const tokens = rows.reduce(tokenRowReducer, {});
+    return Object.values(tokens);
+};
 
 export class ApiTokenStore implements IApiTokenStore {
     private logger: Logger;
@@ -72,26 +97,64 @@ export class ApiTokenStore implements IApiTokenStore {
 
     async getAll(): Promise<IApiToken[]> {
         const stopTimer = this.timer('getAll');
-        const rows = await this.db<ITokenTable>(TABLE);
+        const rows = await this.makeTokenProjectQuery();
         stopTimer();
-        return rows.map(toToken);
+        return toTokens(rows);
     }
 
     async getAllActive(): Promise<IApiToken[]> {
         const stopTimer = this.timer('getAllActive');
-        const rows = await this.db<ITokenTable>(TABLE)
+        const rows = await this.makeTokenProjectQuery()
             .where('expires_at', 'IS', null)
             .orWhere('expires_at', '>', 'now()');
         stopTimer();
-        return rows.map(toToken);
+        return toTokens(rows);
+    }
+
+    private makeTokenProjectQuery() {
+        return this.db<ITokenRow>(`${TABLE} as tokens`)
+            .leftJoin(
+                `${API_LINK_TABLE} as token_project_link`,
+                'tokens.secret',
+                'token_project_link.secret',
+            )
+            .select(
+                'tokens.secret',
+                'username',
+                'type',
+                'expires_at',
+                'created_at',
+                'seen_at',
+                'environment',
+                'token_project_link.project',
+            );
     }
 
     async insert(newToken: IApiTokenCreate): Promise<IApiToken> {
-        const [row] = await this.db<ITokenTable>(TABLE).insert(
-            toRow(newToken),
-            ['created_at'],
-        );
-        return { ...newToken, createdAt: row.created_at };
+        const response = await this.db.transaction(async (tx) => {
+            const [row] = await tx<ITokenInsert>(TABLE).insert(
+                toRow(newToken),
+                ['created_at'],
+            );
+
+            const updateProjectTasks = (newToken.projects || [])
+                .filter((project) => {
+                    return project !== ALL_PROJECTS;
+                })
+                .map((project) => {
+                    return tx.raw(
+                        `INSERT INTO ${API_LINK_TABLE} VALUES (?, ?)`,
+                        [newToken.secret, project],
+                    );
+                });
+            await Promise.all(updateProjectTasks);
+            return {
+                ...newToken,
+                project: newToken.projects?.join(',') || '*',
+                createdAt: row.created_at,
+            };
+        });
+        return response;
     }
 
     destroy(): void {}
@@ -106,25 +169,25 @@ export class ApiTokenStore implements IApiTokenStore {
     }
 
     async get(key: string): Promise<IApiToken> {
-        const row = await this.db(TABLE).where('secret', key).first();
-        return toToken(row);
+        const row = await this.makeTokenProjectQuery().where('secret', key);
+        return toTokens(row)[0];
     }
 
     async delete(secret: string): Promise<void> {
-        return this.db<ITokenTable>(TABLE).where({ secret }).del();
+        return this.db<ITokenRow>(TABLE).where({ secret }).del();
     }
 
     async deleteAll(): Promise<void> {
-        return this.db<ITokenTable>(TABLE).del();
+        return this.db<ITokenRow>(TABLE).del();
     }
 
     async setExpiry(secret: string, expiresAt: Date): Promise<IApiToken> {
-        const rows = await this.db<ITokenTable>(TABLE)
+        const rows = await this.makeTokenProjectQuery()
             .update({ expires_at: expiresAt })
             .where({ secret })
             .returning('*');
         if (rows.length > 0) {
-            return toToken(rows[0]);
+            return toTokens(rows)[0];
         }
         throw new NotFoundError('Could not find api-token.');
     }

src/lib/routes/client-api/feature.ts

@@ -10,7 +10,7 @@ import { IFeatureToggleQuery } from '../../types/model';
 import NotFoundError from '../../error/notfound-error';
 import { IAuthRequest } from '../unleash-types';
 import ApiUser from '../../types/api-user';
-import { ALL } from '../../types/models/api-token';
+import { ALL, isAllProjects } from '../../types/models/api-token';
 
 const version = 2;
 
@@ -65,8 +65,8 @@ export default class FeatureController extends Controller {
 
         const override: QueryOverride = {};
         if (user instanceof ApiUser) {
-            if (user.project !== ALL) {
-                override.project = [user.project];
+            if (!isAllProjects(user.projects)) {
+                override.project = user.projects;
             }
             if (user.environment !== ALL) {
                 override.environment = user.environment;

src/lib/schema/api-token-schema.test.ts

@@ -0,0 +1,44 @@
+import { ALL } from '../types/models/api-token';
+import { createApiToken } from './api-token-schema';
+
+test('should reject token with projects and project', async () => {
+    expect.assertions(1);
+    try {
+        await createApiToken.validateAsync({
+            username: 'test',
+            type: 'admin',
+            project: 'default',
+            projects: ['default'],
+        });
+    } catch (error) {
+        expect(error.details[0].message).toEqual(
+            '"project" must not exist simultaneously with [projects]',
+        );
+    }
+});
+
+test('should not have default project set if projects is present', async () => {
+    let token = await createApiToken.validateAsync({
+        username: 'test',
+        type: 'admin',
+        projects: ['default'],
+    });
+    expect(token.project).not.toBeDefined();
+});
+
+test('should have project set to default if projects is missing', async () => {
+    let token = await createApiToken.validateAsync({
+        username: 'test',
+        type: 'admin',
+    });
+    expect(token.project).toBe(ALL);
+});
+
+test('should not have projects set if project is present', async () => {
+    let token = await createApiToken.validateAsync({
+        username: 'test',
+        type: 'admin',
+        project: 'default',
+    });
+    expect(token.projects).not.toBeDefined();
+});

src/lib/schema/api-token-schema.ts

@@ -12,11 +12,16 @@ export const createApiToken = joi
             .required()
             .valid(ApiTokenType.ADMIN, ApiTokenType.CLIENT),
         expiresAt: joi.date().optional(),
-        project: joi.string().optional().default(ALL),
+        project: joi.when('projects', {
+            not: joi.required(),
+            then: joi.string().optional().default(ALL),
+        }),
+        projects: joi.array().min(0).optional(),
         environment: joi.when('type', {
             is: joi.string().valid(ApiTokenType.CLIENT),
             then: joi.string().optional().default(DEFAULT_ENV),
             otherwise: joi.string().optional().default(ALL),
         }),
     })
+    .nand('project', 'projects')
     .options({ stripUnknown: true, allowUnknown: false, abortEarly: false });

src/lib/services/api-token-service.ts

@@ -7,9 +7,12 @@ import ApiUser from '../types/api-user';
 import {
     ApiTokenType,
     IApiToken,
+    ILegacyApiTokenCreate,
     IApiTokenCreate,
     validateApiToken,
     validateApiTokenEnvironment,
+    mapLegacyToken,
+    mapLegacyTokenWithSecret,
 } from '../types/models/api-token';
 import { IApiTokenStore } from '../types/stores/api-token-store';
 import { FOREIGN_KEY_VIOLATION } from '../error/db-error';
@@ -67,13 +70,15 @@ export class ApiTokenService {
         return this.store.getAllActive();
     }
 
-    private async initApiTokens(tokens: IApiTokenCreate[]) {
+    private async initApiTokens(tokens: ILegacyApiTokenCreate[]) {
         const tokenCount = await this.store.count();
         if (tokenCount > 0) {
             return;
         }
         try {
-            const createAll = tokens.map((t) => this.insertNewApiToken(t));
+            const createAll = tokens
+                .map(mapLegacyTokenWithSecret)
+                .map((t) => this.insertNewApiToken(t));
             await Promise.all(createAll);
         } catch (e) {
             this.logger.error('Unable to create initial Admin API tokens');
@@ -89,7 +94,7 @@ export class ApiTokenService {
             return new ApiUser({
                 username: token.username,
                 permissions,
-                project: token.project,
+                projects: token.projects,
                 environment: token.environment,
                 type: token.type,
             });
@@ -108,7 +113,17 @@ export class ApiTokenService {
         return this.store.delete(secret);
     }
 
+    /**
+     * @deprecated This may be removed in a future release, prefer createApiTokenWithProjects
+     */
     public async createApiToken(
+        newToken: Omit<ILegacyApiTokenCreate, 'secret'>,
+    ): Promise<IApiToken> {
+        const token = mapLegacyToken(newToken);
+        return this.createApiTokenWithProjects(token);
+    }
+
+    public async createApiTokenWithProjects(
         newToken: Omit<IApiTokenCreate, 'secret'>,
     ): Promise<IApiToken> {
         validateApiToken(newToken);
@@ -131,8 +146,11 @@ export class ApiTokenService {
         } catch (error) {
             if (error.code === FOREIGN_KEY_VIOLATION) {
                 let { message } = error;
-                if (error.constraint === 'api_tokens_project_fkey') {
-                    message = `Project=${newApiToken.project} does not exist`;
+                if (error.constraint === 'api_token_project_project_fkey') {
+                    message = `Project=${this.findInvalidProject(
+                        error.detail,
+                        newApiToken.projects,
+                    )} does not exist`;
                 } else if (error.constraint === 'api_tokens_environment_fkey') {
                     message = `Environment=${newApiToken.environment} does not exist`;
                 }
@@ -142,9 +160,23 @@ export class ApiTokenService {
         }
     }
 
-    private generateSecretKey({ project, environment }) {
+    private findInvalidProject(errorDetails, projects) {
+        if (!errorDetails) {
+            return 'invalid';
+        }
+        let invalidProject = projects.find((project) => {
+            return errorDetails.includes(`=(${project})`);
+        });
+        return invalidProject || 'invalid';
+    }
+
+    private generateSecretKey({ projects, environment }) {
         const randomStr = crypto.randomBytes(28).toString('hex');
-        return `${project}:${environment}.${randomStr}`;
+        if (projects.length > 1) {
+            return `[]:${environment}.${randomStr}`;
+        } else {
+            return `${projects[0]}:${environment}.${randomStr}`;
+        }
     }
 
     destroy(): void {