From 66d9d7a6d25772a070bb32d1d1fbdd0af9664b67 Mon Sep 17 00:00:00 2001 From: olav Date: Tue, 29 Mar 2022 14:59:14 +0200 Subject: [PATCH] feat: add segments (#1426) * refactor: fix missing tsconfig path in .eslintrc * refactor: require contextName and operator * refactor: fix crash on missing feature strategies * feat: add segments schema * feat: add segments client API * feat: add segments permissions * refactor: fail migration if things exist * refactor: remove strategy IDs from responses * refactor: allow empty description * refactor: add segment import/export * refactor: add perf scripts * refactor: add get segment fn * refactor: move constraint validation endpoint * refactor: use a separate id for segment updates * refactor: use PERF_AUTH_KEY for artillery * refactor: adjust segment seed size * refactor: add missing event data await * refactor: improve method order * refactor: remove request body limit override --- package.json | 2 + perf/README.md | 32 +++ perf/env.sh | 4 + perf/seed/.gitignore | 1 + perf/seed/export.sh | 12 ++ perf/seed/import.sh | 13 ++ perf/test/.gitignore | 2 + perf/test/artillery.sh | 13 ++ perf/test/artillery.yaml | 12 ++ perf/test/gzip.sh | 25 +++ src/lib/db/feature-strategy-store.ts | 29 ++- src/lib/db/feature-toggle-client-store.ts | 167 ++++++++++----- src/lib/db/index.ts | 3 + src/lib/db/segment-store.ts | 193 ++++++++++++++++++ src/lib/experimental.ts | 23 +++ src/lib/routes/admin-api/constraints.ts | 35 ++++ src/lib/routes/admin-api/feature.ts | 2 +- src/lib/routes/admin-api/index.ts | 5 + src/lib/routes/admin-api/project/features.ts | 20 +- src/lib/routes/client-api/index.ts | 9 + src/lib/routes/client-api/segments.ts | 29 +++ src/lib/schema/feature-schema.ts | 7 +- src/lib/services/index.ts | 3 + src/lib/services/segment-schema.ts | 19 ++ src/lib/services/segment-service.ts | 107 ++++++++++ src/lib/services/state-schema.ts | 6 + src/lib/services/state-service.ts | 63 +++++- src/lib/types/events.ts | 3 + src/lib/types/model.ts | 17 ++ src/lib/types/option.ts | 9 +- src/lib/types/partial.ts | 10 + src/lib/types/permissions.ts | 3 + src/lib/types/services.ts | 2 + src/lib/types/stores.ts | 2 + .../types/stores/feature-strategies-store.ts | 2 +- src/lib/types/stores/segment-store.ts | 26 +++ src/lib/util/collect-ids.ts | 3 + src/lib/util/random-id.ts | 5 + src/migrations/.eslintrc | 17 +- src/migrations/20220307130902-add-segments.js | 73 +++++++ src/server-dev.ts | 9 +- src/test/config/test-config.ts | 4 + .../e2e/api/admin/constraints.e2e.test.ts | 36 ++++ src/test/e2e/api/admin/state.e2e.test.ts | 16 ++ src/test/e2e/api/client/segment.e2e.test.ts | 147 +++++++++++++ src/test/e2e/seed/segment.seed.ts | 149 ++++++++++++++ src/test/examples/exported-segments.json | 77 +++++++ .../fixtures/fake-feature-strategies-store.ts | 4 + src/test/fixtures/fake-segment-store.ts | 54 +++++ src/test/fixtures/store.ts | 2 + 50 files changed, 1408 insertions(+), 98 deletions(-) create mode 100644 perf/README.md create mode 100644 perf/env.sh create mode 100644 perf/seed/.gitignore create mode 100755 perf/seed/export.sh create mode 100755 perf/seed/import.sh create mode 100644 perf/test/.gitignore create mode 100755 perf/test/artillery.sh create mode 100644 perf/test/artillery.yaml create mode 100755 perf/test/gzip.sh create mode 100644 src/lib/db/segment-store.ts create mode 100644 src/lib/experimental.ts create mode 100644 src/lib/routes/admin-api/constraints.ts create mode 100644 src/lib/routes/client-api/segments.ts create mode 100644 src/lib/services/segment-schema.ts create mode 100644 src/lib/services/segment-service.ts create mode 100644 src/lib/types/partial.ts create mode 100644 src/lib/types/stores/segment-store.ts create mode 100644 src/lib/util/collect-ids.ts create mode 100644 src/lib/util/random-id.ts create mode 100644 src/migrations/20220307130902-add-segments.js create mode 100644 src/test/e2e/api/admin/constraints.e2e.test.ts create mode 100644 src/test/e2e/api/client/segment.e2e.test.ts create mode 100644 src/test/e2e/seed/segment.seed.ts create mode 100644 src/test/examples/exported-segments.json create mode 100644 src/test/fixtures/fake-segment-store.ts diff --git a/package.json b/package.json index 3898923917e..b58d4c23d8f 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,8 @@ "test:watch": "yarn test --watch", "test:coverage": "NODE_ENV=test PORT=4243 jest --coverage --forceExit --testTimeout=10000", "test:coverage:jest": "NODE_ENV=test PORT=4243 jest --silent --ci --json --coverage --testLocationInResults --outputFile=\"report.json\" --forceExit --testTimeout=10000", + "seed:setup": "ts-node src/test/e2e/seed/segment.seed.ts", + "seed:serve": "UNLEASH_DATABASE_NAME=unleash_test UNLEASH_DATABASE_SCHEMA=seed yarn run start:dev", "clean": "del-cli --force dist" }, "jest": { diff --git a/perf/README.md b/perf/README.md new file mode 100644 index 00000000000..ad01262c72a --- /dev/null +++ b/perf/README.md @@ -0,0 +1,32 @@ +# /perf + +Testing performance testing! Files of note: + +```shell +# Configure the app URL and auth token to use in performance testing. +./env.sh + +# Export all the data from the app at the configured URL. +./seed/export.sh + +# Import previously exported data to the app instance. +./seed/import.sh + +# Measure the GZIP response size for interesting endpoints. +./test/gzip.sh + +# Run a few load test scenarios against the app. +./test/artillery.sh +``` + +See also the following scripts in `package.json`: + +```shell +# Fill the unleash_testing/seed schema with seed data. +$ yarn seed:setup + +# Serve the unleash_testing/seed schema data, for exports. +$ yarn seed:serve +``` + +Edit files in `/test/e2e/seed` to change the amount data. diff --git a/perf/env.sh b/perf/env.sh new file mode 100644 index 00000000000..b6ff97d4fd1 --- /dev/null +++ b/perf/env.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +export PERF_AUTH_KEY="*:*.964a287e1b728cb5f4f3e0120df92cb5" +export PERF_APP_URL="http://localhost:4242" diff --git a/perf/seed/.gitignore b/perf/seed/.gitignore new file mode 100644 index 00000000000..4df8098eac9 --- /dev/null +++ b/perf/seed/.gitignore @@ -0,0 +1 @@ +/export.json diff --git a/perf/seed/export.sh b/perf/seed/export.sh new file mode 100755 index 00000000000..4bace24222d --- /dev/null +++ b/perf/seed/export.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -feu + +cd "$(dirname "$0")" + +. ../env.sh + +# Export data. Delete environments since they can't be imported. +curl -H "Authorization: $PERF_AUTH_KEY" "$PERF_APP_URL/api/admin/state/export" \ + | jq 'del(.environments)' \ + > export.json diff --git a/perf/seed/import.sh b/perf/seed/import.sh new file mode 100755 index 00000000000..0fe59b703a0 --- /dev/null +++ b/perf/seed/import.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -feu + +cd "$(dirname "$0")" + +. ../env.sh + +curl -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: $PERF_AUTH_KEY" \ + -d @export.json \ + "$PERF_APP_URL/api/admin/state/import?drop=true&keep=false" diff --git a/perf/test/.gitignore b/perf/test/.gitignore new file mode 100644 index 00000000000..2275e0aad64 --- /dev/null +++ b/perf/test/.gitignore @@ -0,0 +1,2 @@ +/artillery.json +/artillery.json.html diff --git a/perf/test/artillery.sh b/perf/test/artillery.sh new file mode 100755 index 00000000000..87655b82390 --- /dev/null +++ b/perf/test/artillery.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -feu + +cd "$(dirname "$0")" + +. ../env.sh + +artillery run ./artillery.yaml --output artillery.json + +artillery report artillery.json + +echo "See artillery.json.html for results" diff --git a/perf/test/artillery.yaml b/perf/test/artillery.yaml new file mode 100644 index 00000000000..fef84bebdbf --- /dev/null +++ b/perf/test/artillery.yaml @@ -0,0 +1,12 @@ +config: + target: "http://localhost:4242" + defaults: + headers: + authorization: "{{ $processEnvironment.PERF_AUTH_KEY }}" + phases: + - duration: 60 + arrivalRate: 10 +scenarios: + - flow: + - get: + url: "/api/client/features" diff --git a/perf/test/gzip.sh b/perf/test/gzip.sh new file mode 100755 index 00000000000..77405b0412b --- /dev/null +++ b/perf/test/gzip.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -feu + +cd "$(dirname "$0")" + +. ../env.sh + +print_response_size () { + local URL + local RES + URL="$1" + RES="$(curl -s -H "Authorization: $PERF_AUTH_KEY" "$URL")" + echo + echo "$URL" + echo + echo "* Byte size: $(echo "$RES" | wc -c) bytes" + echo "* GZIP size: $(echo "$RES" | gzip -6 | wc -c) bytes" +} + +print_response_size "$PERF_APP_URL/api/admin/projects" + +print_response_size "$PERF_APP_URL/api/admin/features" + +print_response_size "$PERF_APP_URL/api/client/features" diff --git a/src/lib/db/feature-strategy-store.ts b/src/lib/db/feature-strategy-store.ts index 54dd709b0a5..69210d30d9c 100644 --- a/src/lib/db/feature-strategy-store.ts +++ b/src/lib/db/feature-strategy-store.ts @@ -14,6 +14,7 @@ import { IStrategyConfig, } from '../types/model'; import { IFeatureStrategiesStore } from '../types/stores/feature-strategies-store'; +import { PartialSome } from '../types/partial'; const COLUMNS = [ 'id', @@ -36,6 +37,7 @@ const mapperToColumnNames = { const T = { features: 'features', featureStrategies: 'feature_strategies', + featureStrategySegment: 'feature_strategy_segment', featureEnvs: 'feature_environments', }; @@ -128,7 +130,7 @@ class FeatureStrategiesStore implements IFeatureStrategiesStore { async exists(key: string): Promise { const result = await this.db.raw( - `SELECT EXISTS (SELECT 1 FROM ${T.featureStrategies} WHERE id = ?) AS present`, + `SELECT EXISTS(SELECT 1 FROM ${T.featureStrategies} WHERE id = ?) AS present`, [key], ); const { present } = result.rows[0]; @@ -148,9 +150,9 @@ class FeatureStrategiesStore implements IFeatureStrategiesStore { } async createStrategyFeatureEnv( - strategyConfig: Omit, + strategyConfig: PartialSome, ): Promise { - const strategyRow = mapInput({ ...strategyConfig, id: uuidv4() }); + const strategyRow = mapInput({ id: uuidv4(), ...strategyConfig }); const rows = await this.db(T.featureStrategies) .insert(strategyRow) .returning('*'); @@ -422,6 +424,27 @@ class FeatureStrategiesStore implements IFeatureStrategiesStore { .where({ feature_name: featureName }) .update({ project_name: newProjectId }); } + + async getStrategiesBySegment( + segmentId: number, + ): Promise { + const stopTimer = this.timer('getStrategiesBySegment'); + const rows = await this.db + .select(this.prefixColumns()) + .from(T.featureStrategies) + .join( + T.featureStrategySegment, + `${T.featureStrategySegment}.feature_strategy_id`, + `${T.featureStrategies}.id`, + ) + .where(`${T.featureStrategySegment}.segment_id`, '=', segmentId); + stopTimer(); + return rows.map(mapRow); + } + + prefixColumns(): string[] { + return COLUMNS.map((c) => `${T.featureStrategies}.${c}`); + } } module.exports = FeatureStrategiesStore; diff --git a/src/lib/db/feature-toggle-client-store.ts b/src/lib/db/feature-toggle-client-store.ts index 14dfe4843b7..6cbac56119b 100644 --- a/src/lib/db/feature-toggle-client-store.ts +++ b/src/lib/db/feature-toggle-client-store.ts @@ -1,5 +1,4 @@ import { Knex } from 'knex'; -import EventEmitter from 'events'; import metricsHelper from '../util/metrics-helper'; import { DB_TIME } from '../metric-events'; import { Logger, LogProvider } from '../logger'; @@ -10,6 +9,9 @@ import { } from '../types/model'; import { IFeatureToggleClientStore } from '../types/stores/feature-toggle-client-store'; import { DEFAULT_ENV } from '../util/constants'; +import { PartialDeep } from '../types/partial'; +import { EventEmitter } from 'stream'; +import { IExperimentalOptions } from '../experimental'; export interface FeaturesTable { name: string; @@ -29,11 +31,19 @@ export default class FeatureToggleClientStore private logger: Logger; + private experimental: IExperimentalOptions; + private timer: Function; - constructor(db: Knex, eventBus: EventEmitter, getLogger: LogProvider) { + constructor( + db: Knex, + eventBus: EventEmitter, + getLogger: LogProvider, + experimental: IExperimentalOptions, + ) { this.db = db; this.logger = getLogger('feature-toggle-client-store.ts'); + this.experimental = experimental; this.timer = (action) => metricsHelper.wrapTimer(eventBus, DB_TIME, { store: 'feature-toggle', @@ -41,25 +51,6 @@ export default class FeatureToggleClientStore }); } - private getAdminStrategy( - r: any, - includeId: boolean = true, - ): IStrategyConfig { - if (includeId) { - return { - name: r.strategy_name, - constraints: r.constraints || [], - parameters: r.parameters, - id: r.strategy_id, - }; - } - return { - name: r.strategy_name, - constraints: r.constraints || [], - parameters: r.parameters, - }; - } - private async getAll( featureQuery?: IFeatureToggleQuery, archived: boolean = false, @@ -67,24 +58,38 @@ export default class FeatureToggleClientStore ): Promise { const environment = featureQuery?.environment || DEFAULT_ENV; const stopTimer = this.timer('getFeatureAdmin'); + + const { inlineSegmentConstraints = false } = + this.experimental?.segments ?? {}; + + let selectColumns = [ + 'features.name as name', + 'features.description as description', + 'features.type as type', + 'features.project as project', + 'features.stale as stale', + 'features.impression_data as impression_data', + 'features.variants as variants', + 'features.created_at as created_at', + 'features.last_seen_at as last_seen_at', + 'fe.enabled as enabled', + 'fe.environment as environment', + 'fs.id as strategy_id', + 'fs.strategy_name as strategy_name', + 'fs.parameters as parameters', + 'fs.constraints as constraints', + ]; + + if (inlineSegmentConstraints) { + selectColumns = [ + ...selectColumns, + 'segments.id as segment_id', + 'segments.constraints as segment_constraints', + ]; + } + let query = this.db('features') - .select( - 'features.name as name', - 'features.description as description', - 'features.type as type', - 'features.project as project', - 'features.stale as stale', - 'features.impression_data as impression_data', - 'features.variants as variants', - 'features.created_at as created_at', - 'features.last_seen_at as last_seen_at', - 'fe.enabled as enabled', - 'fe.environment as environment', - 'fs.id as strategy_id', - 'fs.strategy_name as strategy_name', - 'fs.parameters as parameters', - 'fs.constraints as constraints', - ) + .select(selectColumns) .fullOuterJoin( this.db('feature_strategies') .select('*') @@ -100,8 +105,21 @@ export default class FeatureToggleClientStore .as('fe'), 'fe.feature_name', 'features.name', - ) - .where({ archived }); + ); + + if (inlineSegmentConstraints) { + query = query + .fullOuterJoin( + 'feature_strategy_segment as fss', + `fss.feature_strategy_id`, + `fs.id`, + ) + .fullOuterJoin('segments', `segments.id`, `fss.segment_id`); + } + + query = query.where({ + archived, + }); if (featureQuery) { if (featureQuery.tag) { @@ -109,14 +127,14 @@ export default class FeatureToggleClientStore .from('feature_tag') .select('feature_name') .whereIn(['tag_type', 'tag_value'], featureQuery.tag); - query = query.whereIn('name', tagQuery); + query = query.whereIn('features.name', tagQuery); } if (featureQuery.project) { query = query.whereIn('project', featureQuery.project); } if (featureQuery.namePrefix) { query = query.where( - 'name', + 'features.name', 'like', `${featureQuery.namePrefix}%`, ); @@ -125,18 +143,16 @@ export default class FeatureToggleClientStore const rows = await query; stopTimer(); + const featureToggles = rows.reduce((acc, r) => { - let feature; - if (acc[r.name]) { - feature = acc[r.name]; - } else { - feature = {}; - } - if (!feature.strategies) { - feature.strategies = []; + let feature: PartialDeep = acc[r.name] ?? { + strategies: [], + }; + if (this.isUnseenStrategyRow(feature, r)) { + feature.strategies.push(this.rowToStrategy(r)); } - if (r.strategy_name) { - feature.strategies.push(this.getAdminStrategy(r, isAdmin)); + if (inlineSegmentConstraints && r.segment_id) { + this.addSegmentToStrategy(feature, r); } feature.impressionData = r.impression_data; feature.enabled = !!r.enabled; @@ -154,7 +170,52 @@ export default class FeatureToggleClientStore acc[r.name] = feature; return acc; }, {}); - return Object.values(featureToggles); + + const features: IFeatureToggleClient[] = Object.values(featureToggles); + + if (!isAdmin) { + // We should not send strategy IDs from the client API, + // as this breaks old versions of the Go SDK (at least). + this.removeIdsFromStrategies(features); + } + + return features; + } + + private rowToStrategy(row: Record): IStrategyConfig { + return { + id: row.strategy_id, + name: row.strategy_name, + constraints: row.constraints || [], + parameters: row.parameters, + }; + } + + private removeIdsFromStrategies(features: IFeatureToggleClient[]) { + features.forEach((feature) => { + feature.strategies.forEach((strategy) => { + delete strategy.id; + }); + }); + } + + private isUnseenStrategyRow( + feature: PartialDeep, + row: Record, + ): boolean { + return ( + row.strategy_id && + !feature.strategies.find((s) => s.id === row.strategy_id) + ); + } + + private addSegmentToStrategy( + feature: PartialDeep, + row: Record, + ) { + feature.strategies + .find((s) => s.id === row.strategy_id) + ?.constraints.push(...row.segment_constraints); } async getClient( diff --git a/src/lib/db/index.ts b/src/lib/db/index.ts index 8d67f87a936..07e7b1106bd 100644 --- a/src/lib/db/index.ts +++ b/src/lib/db/index.ts @@ -28,6 +28,7 @@ import { FeatureEnvironmentStore } from './feature-environment-store'; import { ClientMetricsStoreV2 } from './client-metrics-store-v2'; import UserSplashStore from './user-splash-store'; import RoleStore from './role-store'; +import SegmentStore from './segment-store'; export const createStores = ( config: IUnleashConfig, @@ -69,6 +70,7 @@ export const createStores = ( db, eventBus, getLogger, + config.experimental, ), environmentStore: new EnvironmentStore(db, eventBus, getLogger), featureTagStore: new FeatureTagStore(db, eventBus, getLogger), @@ -79,6 +81,7 @@ export const createStores = ( ), userSplashStore: new UserSplashStore(db, eventBus, getLogger), roleStore: new RoleStore(db, eventBus, getLogger), + segmentStore: new SegmentStore(db, eventBus, getLogger), }; }; diff --git a/src/lib/db/segment-store.ts b/src/lib/db/segment-store.ts new file mode 100644 index 00000000000..cfc02313428 --- /dev/null +++ b/src/lib/db/segment-store.ts @@ -0,0 +1,193 @@ +import { ISegmentStore } from '../types/stores/segment-store'; +import { IConstraint, IFeatureStrategySegment, ISegment } from '../types/model'; +import { Logger, LogProvider } from '../logger'; +import { Knex } from 'knex'; +import EventEmitter from 'events'; +import NotFoundError from '../error/notfound-error'; +import User from '../types/user'; +import { PartialSome } from '../types/partial'; + +const T = { + segments: 'segments', + featureStrategies: 'feature_strategies', + featureStrategySegment: 'feature_strategy_segment', +}; + +const COLUMNS = [ + 'id', + 'name', + 'description', + 'created_by', + 'created_at', + 'constraints', +]; + +interface ISegmentRow { + id: number; + name: string; + description?: string; + created_by?: string; + created_at?: Date; + constraints: IConstraint[]; +} + +interface IFeatureStrategySegmentRow { + feature_strategy_id: string; + segment_id: number; + created_at?: Date; +} + +export default class SegmentStore implements ISegmentStore { + private logger: Logger; + + private eventBus: EventEmitter; + + private db: Knex; + + constructor(db: Knex, eventBus: EventEmitter, getLogger: LogProvider) { + this.db = db; + this.eventBus = eventBus; + this.logger = getLogger('lib/db/segment-store.ts'); + } + + async create( + segment: PartialSome, + user: Partial>, + ): Promise { + const rows = await this.db(T.segments) + .insert({ + id: segment.id, + name: segment.name, + description: segment.description, + constraints: JSON.stringify(segment.constraints), + created_by: user.username || user.email, + }) + .returning(COLUMNS); + + return this.mapRow(rows[0]); + } + + async update(id: number, segment: Omit): Promise { + const rows = await this.db(T.segments) + .where({ id }) + .update({ + name: segment.name, + description: segment.description, + constraints: JSON.stringify(segment.constraints), + }) + .returning(COLUMNS); + + return this.mapRow(rows[0]); + } + + delete(id: number): Promise { + return this.db(T.segments).where({ id }).del(); + } + + async getAll(): Promise { + const rows: ISegmentRow[] = await this.db + .select(this.prefixColumns()) + .from(T.segments) + .orderBy('name', 'asc'); + + return rows.map(this.mapRow); + } + + async getActive(): Promise { + const rows: ISegmentRow[] = await this.db + .distinct(this.prefixColumns()) + .from(T.segments) + .orderBy('name', 'asc') + .join( + T.featureStrategySegment, + `${T.featureStrategySegment}.segment_id`, + `${T.segments}.id`, + ); + + return rows.map(this.mapRow); + } + + async getByStrategy(strategyId: string): Promise { + const rows = await this.db + .select(this.prefixColumns()) + .from(T.segments) + .join( + T.featureStrategySegment, + `${T.featureStrategySegment}.segment_id`, + `${T.segments}.id`, + ) + .where( + `${T.featureStrategySegment}.feature_strategy_id`, + '=', + strategyId, + ); + return rows.map(this.mapRow); + } + + deleteAll(): Promise { + return this.db(T.segments).del(); + } + + async exists(id: number): Promise { + const result = await this.db.raw( + `SELECT EXISTS(SELECT 1 FROM ${T.segments} WHERE id = ?) AS present`, + [id], + ); + + return result.rows[0].present; + } + + async get(id: number): Promise { + const rows: ISegmentRow[] = await this.db + .select(this.prefixColumns()) + .from(T.segments) + .where({ id }); + + return this.mapRow(rows[0]); + } + + async addToStrategy(id: number, strategyId: string): Promise { + await this.db(T.featureStrategySegment).insert({ + segment_id: id, + feature_strategy_id: strategyId, + }); + } + + async removeFromStrategy(id: number, strategyId: string): Promise { + await this.db(T.featureStrategySegment) + .where({ segment_id: id, feature_strategy_id: strategyId }) + .del(); + } + + async getAllFeatureStrategySegments(): Promise { + const rows: IFeatureStrategySegmentRow[] = await this.db + .select(['segment_id', 'feature_strategy_id']) + .from(T.featureStrategySegment); + + return rows.map((row) => ({ + featureStrategyId: row.feature_strategy_id, + segmentId: row.segment_id, + })); + } + + prefixColumns(): string[] { + return COLUMNS.map((c) => `${T.segments}.${c}`); + } + + mapRow(row?: ISegmentRow): ISegment { + if (!row) { + throw new NotFoundError('No row'); + } + + return { + id: row.id, + name: row.name, + description: row.description, + constraints: row.constraints, + createdBy: row.created_by, + createdAt: row.created_at, + }; + } + + destroy(): void {} +} diff --git a/src/lib/experimental.ts b/src/lib/experimental.ts new file mode 100644 index 00000000000..5a4934c023a --- /dev/null +++ b/src/lib/experimental.ts @@ -0,0 +1,23 @@ +export interface IExperimentalOptions { + metricsV2?: IExperimentalToggle; + clientFeatureMemoize?: IExperimentalToggle; + segments?: IExperimentalSegments; +} + +export interface IExperimentalToggle { + enabled: boolean; +} + +export interface IExperimentalSegments { + enableSegmentsClientApi: boolean; + enableSegmentsAdminApi: boolean; + inlineSegmentConstraints: boolean; +} + +export const experimentalSegmentsConfig = (): IExperimentalSegments => { + return { + enableSegmentsAdminApi: true, + enableSegmentsClientApi: true, + inlineSegmentConstraints: true, + }; +}; diff --git a/src/lib/routes/admin-api/constraints.ts b/src/lib/routes/admin-api/constraints.ts new file mode 100644 index 00000000000..2c2b056d29e --- /dev/null +++ b/src/lib/routes/admin-api/constraints.ts @@ -0,0 +1,35 @@ +import { Request, Response } from 'express'; +import FeatureToggleService from '../../services/feature-toggle-service'; +import { IUnleashConfig } from '../../types/option'; +import { IUnleashServices } from '../../types'; +import { IConstraint } from '../../types/model'; +import { NONE } from '../../types/permissions'; +import Controller from '../controller'; +import { Logger } from '../../logger'; + +export default class ConstraintController extends Controller { + private featureService: FeatureToggleService; + + private readonly logger: Logger; + + constructor( + config: IUnleashConfig, + { + featureToggleServiceV2, + }: Pick, + ) { + super(config); + this.featureService = featureToggleServiceV2; + this.logger = config.getLogger('/admin-api/validation.ts'); + + this.post('/validate', this.validateConstraint, NONE); + } + + async validateConstraint( + req: Request<{}, undefined, IConstraint>, + res: Response, + ): Promise { + await this.featureService.validateConstraint(req.body); + res.status(204).send(); + } +} diff --git a/src/lib/routes/admin-api/feature.ts b/src/lib/routes/admin-api/feature.ts index 1ccc4ed066e..5a5689af048 100644 --- a/src/lib/routes/admin-api/feature.ts +++ b/src/lib/routes/admin-api/feature.ts @@ -157,7 +157,7 @@ class FeatureController extends Controller { true, ); const strategies = await Promise.all( - toggle.strategies.map(async (s) => + (toggle.strategies ?? []).map(async (s) => this.service.createStrategy( s, { diff --git a/src/lib/routes/admin-api/index.ts b/src/lib/routes/admin-api/index.ts index baaab648804..79e7b415d04 100644 --- a/src/lib/routes/admin-api/index.ts +++ b/src/lib/routes/admin-api/index.ts @@ -24,6 +24,7 @@ import UserFeedbackController from './user-feedback-controller'; import UserSplashController from './user-splash-controller'; import ProjectApi from './project'; import { EnvironmentsController } from './environments-controller'; +import ConstraintsController from './constraints'; class AdminApi extends Controller { constructor(config: IUnleashConfig, services: IUnleashServices) { @@ -99,6 +100,10 @@ class AdminApi extends Controller { '/splash', new UserSplashController(config, services).router, ); + this.app.use( + '/constraints', + new ConstraintsController(config, services).router, + ); } index(req, res) { diff --git a/src/lib/routes/admin-api/project/features.ts b/src/lib/routes/admin-api/project/features.ts index 14f0390b7c0..2de059b2c8a 100644 --- a/src/lib/routes/admin-api/project/features.ts +++ b/src/lib/routes/admin-api/project/features.ts @@ -10,7 +10,6 @@ import { CREATE_FEATURE_STRATEGY, DELETE_FEATURE, DELETE_FEATURE_STRATEGY, - NONE, UPDATE_FEATURE, UPDATE_FEATURE_ENVIRONMENT, UPDATE_FEATURE_STRATEGY, @@ -73,7 +72,6 @@ export default class ProjectFeaturesController extends Controller { this.featureService = featureToggleServiceV2; this.logger = config.getLogger('/admin-api/project/features.ts'); - // Environments this.get(`${PATH_ENV}`, this.getEnvironment); this.post( `${PATH_ENV}/on`, @@ -85,14 +83,13 @@ export default class ProjectFeaturesController extends Controller { this.toggleEnvironmentOff, UPDATE_FEATURE_ENVIRONMENT, ); - // activation strategies + this.get(`${PATH_STRATEGIES}`, this.getStrategies); this.post( `${PATH_STRATEGIES}`, this.addStrategy, CREATE_FEATURE_STRATEGY, ); - this.get(`${PATH_STRATEGY}`, this.getStrategy); this.put( `${PATH_STRATEGY}`, @@ -109,18 +106,10 @@ export default class ProjectFeaturesController extends Controller { this.deleteStrategy, DELETE_FEATURE_STRATEGY, ); - this.post( - `${PATH_FEATURE}/constraint/validate`, - this.validateConstraint, - NONE, - ); - // feature toggles this.get(PATH, this.getFeatures); this.post(PATH, this.createFeature, CREATE_FEATURE); - this.post(PATH_FEATURE_CLONE, this.cloneFeature, CREATE_FEATURE); - this.get(PATH_FEATURE, this.getFeature); this.put(PATH_FEATURE, this.updateFeature, UPDATE_FEATURE); this.patch(PATH_FEATURE, this.patchFeature, UPDATE_FEATURE); @@ -343,13 +332,6 @@ export default class ProjectFeaturesController extends Controller { res.status(200).json(updatedStrategy); } - async validateConstraint(req: Request, res: Response): Promise { - const constraint: IConstraint = { ...req.body }; - - await this.featureService.validateConstraint(constraint); - res.status(204).send(); - } - async getStrategy( req: IAuthRequest, res: Response, diff --git a/src/lib/routes/client-api/index.ts b/src/lib/routes/client-api/index.ts index a5532c330d5..aaa85eed85b 100644 --- a/src/lib/routes/client-api/index.ts +++ b/src/lib/routes/client-api/index.ts @@ -5,16 +5,25 @@ import MetricsController from './metrics'; import RegisterController from './register'; import { IUnleashConfig } from '../../types/option'; import { IUnleashServices } from '../../types'; +import { SegmentsController } from './segments'; const apiDef = require('./api-def.json'); export default class ClientApi extends Controller { constructor(config: IUnleashConfig, services: IUnleashServices) { super(config); + this.get('/', this.index); this.use('/features', new FeatureController(services, config).router); this.use('/metrics', new MetricsController(services, config).router); this.use('/register', new RegisterController(services, config).router); + + if (config.experimental?.segments?.enableSegmentsClientApi) { + this.use( + '/segments', + new SegmentsController(services, config).router, + ); + } } index(req: Request, res: Response): void { diff --git a/src/lib/routes/client-api/segments.ts b/src/lib/routes/client-api/segments.ts new file mode 100644 index 00000000000..aed95ab0f40 --- /dev/null +++ b/src/lib/routes/client-api/segments.ts @@ -0,0 +1,29 @@ +import { Response } from 'express'; +import Controller from '../controller'; +import { IUnleashConfig } from '../../types/option'; +import { IUnleashServices } from '../../types'; +import { Logger } from '../../logger'; +import { SegmentService } from '../../services/segment-service'; +import { IAuthRequest } from '../unleash-types'; + +export class SegmentsController extends Controller { + private logger: Logger; + + private segmentService: SegmentService; + + constructor( + { segmentService }: Pick, + config: IUnleashConfig, + ) { + super(config); + this.logger = config.getLogger('/client-api/segments.ts'); + this.segmentService = segmentService; + + this.get('/active', this.getActive); + } + + async getActive(req: IAuthRequest, res: Response): Promise { + const segments = await this.segmentService.getActive(); + res.json({ segments }); + } +} diff --git a/src/lib/schema/feature-schema.ts b/src/lib/schema/feature-schema.ts index cf6692ed644..035fc53458e 100644 --- a/src/lib/schema/feature-schema.ts +++ b/src/lib/schema/feature-schema.ts @@ -8,8 +8,11 @@ export const nameSchema = joi .options({ stripUnknown: true, allowUnknown: false, abortEarly: false }); export const constraintSchema = joi.object().keys({ - contextName: joi.string(), - operator: joi.string().valid(...ALL_OPERATORS), + contextName: joi.string().required(), + operator: joi + .string() + .valid(...ALL_OPERATORS) + .required(), // Constraints must have a values array to support legacy SDKs. values: joi.array().items(joi.string().min(1).max(100)).default([]), value: joi.optional(), diff --git a/src/lib/services/index.ts b/src/lib/services/index.ts index d2a80e3f5c7..d14209b84e2 100644 --- a/src/lib/services/index.ts +++ b/src/lib/services/index.ts @@ -28,6 +28,7 @@ import EnvironmentService from './environment-service'; import FeatureTagService from './feature-tag-service'; import ProjectHealthService from './project-health-service'; import UserSplashService from './user-splash-service'; +import { SegmentService } from './segment-service'; export const createServices = ( stores: IUnleashStores, @@ -74,6 +75,7 @@ export const createServices = ( featureToggleServiceV2, ); const userSplashService = new UserSplashService(stores, config); + const segmentService = new SegmentService(stores, config); return { accessService, @@ -103,6 +105,7 @@ export const createServices = ( featureTagService, projectHealthService, userSplashService, + segmentService, }; }; diff --git a/src/lib/services/segment-schema.ts b/src/lib/services/segment-schema.ts new file mode 100644 index 00000000000..90adcf95e08 --- /dev/null +++ b/src/lib/services/segment-schema.ts @@ -0,0 +1,19 @@ +import joi from 'joi'; +import { constraintSchema } from '../schema/feature-schema'; + +export const segmentSchema = joi + .object() + .keys({ + name: joi.string().required(), + description: joi.string().allow(null).allow('').optional(), + constraints: joi.array().items(constraintSchema).required(), + }) + .options({ allowUnknown: true }); + +export const featureStrategySegmentSchema = joi + .object() + .keys({ + segmentId: joi.number().required(), + featureStrategyId: joi.string().required(), + }) + .options({ allowUnknown: true }); diff --git a/src/lib/services/segment-service.ts b/src/lib/services/segment-service.ts new file mode 100644 index 00000000000..cccc9e99148 --- /dev/null +++ b/src/lib/services/segment-service.ts @@ -0,0 +1,107 @@ +import { IUnleashConfig } from '../types/option'; +import { IEventStore } from '../types/stores/event-store'; +import { IUnleashStores } from '../types'; +import { Logger } from '../logger'; +import { ISegmentStore } from '../types/stores/segment-store'; +import { IFeatureStrategy, ISegment } from '../types/model'; +import { segmentSchema } from './segment-schema'; +import { + SEGMENT_CREATED, + SEGMENT_DELETED, + SEGMENT_UPDATED, +} from '../types/events'; +import User from '../types/user'; +import { IFeatureStrategiesStore } from '../types/stores/feature-strategies-store'; + +export class SegmentService { + private logger: Logger; + + private segmentStore: ISegmentStore; + + private featureStrategiesStore: IFeatureStrategiesStore; + + private eventStore: IEventStore; + + constructor( + { + segmentStore, + featureStrategiesStore, + eventStore, + }: Pick< + IUnleashStores, + 'segmentStore' | 'featureStrategiesStore' | 'eventStore' + >, + { getLogger }: Pick, + ) { + this.segmentStore = segmentStore; + this.featureStrategiesStore = featureStrategiesStore; + this.eventStore = eventStore; + this.logger = getLogger('services/segment-service.ts'); + } + + async get(id: number): Promise { + return this.segmentStore.get(id); + } + + async getAll(): Promise { + return this.segmentStore.getAll(); + } + + async getActive(): Promise { + return this.segmentStore.getActive(); + } + + // Used by unleash-enterprise. + async getByStrategy(strategyId: string): Promise { + return this.segmentStore.getByStrategy(strategyId); + } + + // Used by unleash-enterprise. + async getStrategies(id: number): Promise { + return this.featureStrategiesStore.getStrategiesBySegment(id); + } + + async create(data: unknown, user: User): Promise { + const input = await segmentSchema.validateAsync(data); + const segment = await this.segmentStore.create(input, user); + + await this.eventStore.store({ + type: SEGMENT_CREATED, + createdBy: user.email || user.username, + data: segment, + }); + } + + async update(id: number, data: unknown, user: User): Promise { + const input = await segmentSchema.validateAsync(data); + const preData = await this.segmentStore.get(id); + const segment = await this.segmentStore.update(id, input); + + await this.eventStore.store({ + type: SEGMENT_UPDATED, + createdBy: user.email || user.username, + data: segment, + preData, + }); + } + + async delete(id: number, user: User): Promise { + const segment = this.segmentStore.get(id); + await this.segmentStore.delete(id); + await this.eventStore.store({ + type: SEGMENT_DELETED, + createdBy: user.email || user.username, + data: segment, + }); + } + + // Used by unleash-enterprise. + async addToStrategy(id: number, strategyId: string): Promise { + await this.segmentStore.addToStrategy(id, strategyId); + } + + // Used by unleash-enterprise. + async removeFromStrategy(id: number, strategyId: string): Promise { + await this.segmentStore.removeFromStrategy(id, strategyId); + } +} diff --git a/src/lib/services/state-schema.ts b/src/lib/services/state-schema.ts index 0e7bd917581..5059cda465c 100644 --- a/src/lib/services/state-schema.ts +++ b/src/lib/services/state-schema.ts @@ -5,6 +5,7 @@ import { tagSchema } from './tag-schema'; import { tagTypeSchema } from './tag-type-schema'; import { projectSchema } from './project-schema'; import { nameType } from '../routes/util'; +import { featureStrategySegmentSchema, segmentSchema } from './segment-schema'; export const featureStrategySchema = joi .object() @@ -56,4 +57,9 @@ export const stateSchema = joi.object().keys({ .optional() .items(featureEnvironmentsSchema), environments: joi.array().optional().items(environmentSchema), + segments: joi.array().optional().items(segmentSchema), + featureStrategySegments: joi + .array() + .optional() + .items(featureStrategySegmentSchema), }); diff --git a/src/lib/services/state-service.ts b/src/lib/services/state-service.ts index 600dbd9e986..f9f2fbf9b1b 100644 --- a/src/lib/services/state-service.ts +++ b/src/lib/services/state-service.ts @@ -22,13 +22,14 @@ import { IUnleashConfig } from '../types/option'; import { FeatureToggle, IEnvironment, - IImportFile, IFeatureEnvironment, IFeatureStrategy, - ITag, IImportData, + IImportFile, IProject, + ISegment, IStrategyConfig, + ITag, } from '../types/model'; import { Logger } from '../logger'; import { @@ -47,6 +48,8 @@ import { IFeatureEnvironmentStore } from '../types/stores/feature-environment-st import { IUnleashStores } from '../types/stores'; import { DEFAULT_ENV } from '../util/constants'; import { GLOBAL_ENV } from '../types/environment'; +import { ISegmentStore } from '../types/stores/segment-store'; +import { PartialSome } from '../types/partial'; export interface IBackupOption { includeFeatureToggles: boolean; @@ -61,6 +64,7 @@ interface IExportIncludeOptions { includeProjects?: boolean; includeTags?: boolean; includeEnvironments?: boolean; + includeSegments?: boolean; } export default class StateService { @@ -86,6 +90,8 @@ export default class StateService { private environmentStore: IEnvironmentStore; + private segmentStore: ISegmentStore; + constructor( stores: IUnleashStores, { getLogger }: Pick, @@ -100,6 +106,7 @@ export default class StateService { this.projectStore = stores.projectStore; this.featureTagStore = stores.featureTagStore; this.environmentStore = stores.environmentStore; + this.segmentStore = stores.segmentStore; this.logger = getLogger('services/state-service.js'); } @@ -227,6 +234,20 @@ export default class StateService { keepExisting, }); } + + if (importData.segments) { + await this.importSegments( + data.segments, + userName, + dropBeforeImport, + ); + } + + if (importData.featureStrategySegments) { + await this.importFeatureStrategySegments( + data.featureStrategySegments, + ); + } } // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types @@ -596,6 +617,35 @@ export default class StateService { } } + async importSegments( + segments: PartialSome[], + userName: string, + dropBeforeImport: boolean, + ): Promise { + if (dropBeforeImport) { + await this.segmentStore.deleteAll(); + } + + await Promise.all( + segments.map((segment) => + this.segmentStore.create(segment, { username: userName }), + ), + ); + } + + async importFeatureStrategySegments( + featureStrategySegments: { + featureStrategyId: string; + segmentId: number; + }[], + ): Promise { + await Promise.all( + featureStrategySegments.map(({ featureStrategyId, segmentId }) => + this.segmentStore.addToStrategy(segmentId, featureStrategyId), + ), + ); + } + // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types async export({ includeFeatureToggles = true, @@ -603,6 +653,7 @@ export default class StateService { includeProjects = true, includeTags = true, includeEnvironments = true, + includeSegments = true, }: IExportIncludeOptions): Promise<{ features: FeatureToggle[]; strategies: IStrategy[]; @@ -639,6 +690,10 @@ export default class StateService { includeFeatureToggles ? this.featureEnvironmentStore.getAll() : Promise.resolve([]), + includeSegments ? this.segmentStore.getAll() : Promise.resolve([]), + includeSegments + ? this.segmentStore.getAllFeatureStrategySegments() + : Promise.resolve([]), ]).then( ([ features, @@ -650,6 +705,8 @@ export default class StateService { featureStrategies, environments, featureEnvironments, + segments, + featureStrategySegments, ]) => ({ version: 3, features, @@ -665,6 +722,8 @@ export default class StateService { featureEnvironments: featureEnvironments.filter((fE) => features.some((f) => fE.featureName === f.name), ), + segments, + featureStrategySegments, }), ); } diff --git a/src/lib/types/events.ts b/src/lib/types/events.ts index 57e700ea49f..1ec7cfbcefd 100644 --- a/src/lib/types/events.ts +++ b/src/lib/types/events.ts @@ -61,6 +61,9 @@ export const USER_UPDATED = 'user-updated'; export const USER_DELETED = 'user-deleted'; export const DROP_ENVIRONMENTS = 'drop-environments'; export const ENVIRONMENT_IMPORT = 'environment-import'; +export const SEGMENT_CREATED = 'segment-created'; +export const SEGMENT_UPDATED = 'segment-updated'; +export const SEGMENT_DELETED = 'segment-deleted'; export const CLIENT_METRICS = 'client-metrics'; diff --git a/src/lib/types/model.ts b/src/lib/types/model.ts index 301b0df1c04..42f26d97a57 100644 --- a/src/lib/types/model.ts +++ b/src/lib/types/model.ts @@ -60,6 +60,9 @@ export interface IFeatureToggleClient { variants: IVariant[]; enabled: boolean; strategies: IStrategyConfig[]; + impressionData?: boolean; + lastSeenAt?: Date; + createdAt?: Date; } export interface IFeatureEnvironmentInfo { @@ -341,3 +344,17 @@ export interface IProjectWithCount extends IProject { featureCount: number; memberCount: number; } + +export interface ISegment { + id: number; + name: string; + description?: string; + constraints: IConstraint[]; + createdBy?: string; + createdAt: Date; +} + +export interface IFeatureStrategySegment { + featureStrategyId: string; + segmentId: number; +} diff --git a/src/lib/types/option.ts b/src/lib/types/option.ts index 61212860e0b..f0d6509fc43 100644 --- a/src/lib/types/option.ts +++ b/src/lib/types/option.ts @@ -1,6 +1,7 @@ import EventEmitter from 'events'; import { LogLevel, LogProvider } from '../logger'; import { IApiTokenCreate } from './models/api-token'; +import { IExperimentalOptions } from '../experimental'; export type EventHook = (eventName: string, data: object) => void; @@ -91,9 +92,7 @@ export interface IUnleashOptions { authentication?: Partial; ui?: object; import?: Partial; - experimental?: { - [key: string]: object; - }; + experimental?: IExperimentalOptions; email?: Partial; secureHeaders?: boolean; enableOAS?: boolean; @@ -146,9 +145,7 @@ export interface IUnleashConfig { authentication: IAuthOption; ui: IUIConfig; import: IImportOption; - experimental: { - [key: string]: any; - }; + experimental?: IExperimentalOptions; email: IEmailOption; secureHeaders: boolean; enableOAS: boolean; diff --git a/src/lib/types/partial.ts b/src/lib/types/partial.ts new file mode 100644 index 00000000000..d00e93cbfc5 --- /dev/null +++ b/src/lib/types/partial.ts @@ -0,0 +1,10 @@ +// Recursively mark all properties as optional. +export type PartialDeep = T extends object + ? { + [P in keyof T]?: PartialDeep; + } + : T; + +// Mark one or more properties as optional. +export type PartialSome = Pick, K> & + Omit; diff --git a/src/lib/types/permissions.ts b/src/lib/types/permissions.ts index 58276a3dabe..ca6bb9744b1 100644 --- a/src/lib/types/permissions.ts +++ b/src/lib/types/permissions.ts @@ -32,3 +32,6 @@ export const UPDATE_TAG_TYPE = 'UPDATE_TAG_TYPE'; export const DELETE_TAG_TYPE = 'DELETE_TAG_TYPE'; export const UPDATE_FEATURE_VARIANTS = 'UPDATE_FEATURE_VARIANTS'; export const MOVE_FEATURE_TOGGLE = 'MOVE_FEATURE_TOGGLE'; +export const CREATE_SEGMENT = 'CREATE_SEGMENT'; +export const UPDATE_SEGMENT = 'UPDATE_SEGMENT'; +export const DELETE_SEGMENT = 'DELETE_SEGMENT'; diff --git a/src/lib/types/services.ts b/src/lib/types/services.ts index 3144ef3822a..d3b382cbb89 100644 --- a/src/lib/types/services.ts +++ b/src/lib/types/services.ts @@ -24,6 +24,7 @@ import FeatureTagService from '../services/feature-tag-service'; import ProjectHealthService from '../services/project-health-service'; import ClientMetricsServiceV2 from '../services/client-metrics/metrics-service-v2'; import UserSplashService from '../services/user-splash-service'; +import { SegmentService } from '../services/segment-service'; export interface IUnleashServices { accessService: AccessService; @@ -53,4 +54,5 @@ export interface IUnleashServices { userService: UserService; versionService: VersionService; userSplashService: UserSplashService; + segmentService: SegmentService; } diff --git a/src/lib/types/stores.ts b/src/lib/types/stores.ts index ea9bac57c7b..f61f9e391db 100644 --- a/src/lib/types/stores.ts +++ b/src/lib/types/stores.ts @@ -24,6 +24,7 @@ import { IFeatureToggleClientStore } from './stores/feature-toggle-client-store' import { IClientMetricsStoreV2 } from './stores/client-metrics-store-v2'; import { IUserSplashStore } from './stores/user-splash-store'; import { IRoleStore } from './stores/role-store'; +import { ISegmentStore } from './stores/segment-store'; export interface IUnleashStores { accessStore: IAccessStore; @@ -52,4 +53,5 @@ export interface IUnleashStores { userStore: IUserStore; userSplashStore: IUserSplashStore; roleStore: IRoleStore; + segmentStore: ISegmentStore; } diff --git a/src/lib/types/stores/feature-strategies-store.ts b/src/lib/types/stores/feature-strategies-store.ts index 85605324ede..28e86a6d4c7 100644 --- a/src/lib/types/stores/feature-strategies-store.ts +++ b/src/lib/types/stores/feature-strategies-store.ts @@ -46,9 +46,9 @@ export interface IFeatureStrategiesStore projectId: String, environment: String, ): Promise; - setProjectForStrategiesBelongingToFeature( featureName: string, newProjectId: string, ): Promise; + getStrategiesBySegment(segmentId: number): Promise; } diff --git a/src/lib/types/stores/segment-store.ts b/src/lib/types/stores/segment-store.ts new file mode 100644 index 00000000000..79da738a9d9 --- /dev/null +++ b/src/lib/types/stores/segment-store.ts @@ -0,0 +1,26 @@ +import { IFeatureStrategySegment, ISegment } from '../model'; +import { Store } from './store'; +import User from '../user'; + +export interface ISegmentStore extends Store { + getAll(): Promise; + + getActive(): Promise; + + getByStrategy(strategyId: string): Promise; + + create( + segment: Omit, + user: Partial>, + ): Promise; + + update(id: number, segment: Omit): Promise; + + delete(id: number): Promise; + + addToStrategy(id: number, strategyId: string): Promise; + + removeFromStrategy(id: number, strategyId: string): Promise; + + getAllFeatureStrategySegments(): Promise; +} diff --git a/src/lib/util/collect-ids.ts b/src/lib/util/collect-ids.ts new file mode 100644 index 00000000000..f1419d51324 --- /dev/null +++ b/src/lib/util/collect-ids.ts @@ -0,0 +1,3 @@ +export const collectIds = (items: { id?: T }[]): T[] => { + return items.map((item) => item.id); +}; diff --git a/src/lib/util/random-id.ts b/src/lib/util/random-id.ts new file mode 100644 index 00000000000..7d9d6bca121 --- /dev/null +++ b/src/lib/util/random-id.ts @@ -0,0 +1,5 @@ +import { v4 as uuidv4 } from 'uuid'; + +export const randomId = (): string => { + return uuidv4(); +}; diff --git a/src/migrations/.eslintrc b/src/migrations/.eslintrc index ad84aa44c36..0719e9f1813 100644 --- a/src/migrations/.eslintrc +++ b/src/migrations/.eslintrc @@ -4,13 +4,16 @@ "parser": "", "plugins": [], "rules": {}, + "settings": {}, + "parserOptions": { + "project": "../../tsconfig.json" + }, "overrides": [ - { - "files": "*.js", - "rules": { - "@typescript-eslint/indent": "off" + { + "files": "*.js", + "rules": { + "@typescript-eslint/indent": "off" + } } - } - ], - "settings": {} + ] } diff --git a/src/migrations/20220307130902-add-segments.js b/src/migrations/20220307130902-add-segments.js new file mode 100644 index 00000000000..96daaf99dd8 --- /dev/null +++ b/src/migrations/20220307130902-add-segments.js @@ -0,0 +1,73 @@ +'use strict'; + +exports.up = function (db, cb) { + db.runSql( + ` + create table segments + ( + id serial primary key, + name text not null, + description text, + created_by text, + created_at timestamp with time zone not null default now(), + constraints jsonb not null default '[]'::jsonb + ); + + create table feature_strategy_segment + ( + feature_strategy_id text not null references feature_strategies (id) on update cascade on delete cascade not null, + segment_id integer not null references segments (id) on update cascade on delete cascade not null, + created_at timestamp with time zone not null default now(), + primary key (feature_strategy_id, segment_id) + ); + + create index feature_strategy_segment_segment_id_index + on feature_strategy_segment (segment_id); + + insert into permissions (permission, display_name, type) values + ('CREATE_SEGMENT', 'Create segments', 'root'), + ('UPDATE_SEGMENT', 'Edit segments', 'root'), + ('DELETE_SEGMENT', 'Delete segments', 'root'); + + insert into role_permission (role_id, permission_id) + select + r.id as role_id, + p.id as permission_id + from roles r + cross join permissions p + where r.name in ( + 'Admin', + 'Editor' + ) + and p.permission in ( + 'CREATE_SEGMENT', + 'UPDATE_SEGMENT', + 'DELETE_SEGMENT' + ); + `, + cb, + ); +}; + +exports.down = function (db, cb) { + db.runSql( + ` + delete from role_permission where permission_id in ( + select id from permissions where permission in ( + 'DELETE_SEGMENT', + 'UPDATE_SEGMENT', + 'CREATE_SEGMENT' + ) + ); + + delete from permissions where permission = 'DELETE_SEGMENT'; + delete from permissions where permission = 'UPDATE_SEGMENT'; + delete from permissions where permission = 'CREATE_SEGMENT'; + + drop index feature_strategy_segment_segment_id_index; + drop table feature_strategy_segment; + drop table segments; + `, + cb, + ); +}; diff --git a/src/server-dev.ts b/src/server-dev.ts index f408dfaec26..45bda849caa 100644 --- a/src/server-dev.ts +++ b/src/server-dev.ts @@ -2,6 +2,7 @@ import { start } from './lib/server-impl'; import { createConfig } from './lib/create-config'; import { LogLevel } from './lib/logger'; import { ApiTokenType } from './lib/types/models/api-token'; +import { experimentalSegmentsConfig } from './lib/experimental'; process.nextTick(async () => { try { @@ -12,7 +13,8 @@ process.nextTick(async () => { password: 'passord', host: 'localhost', port: 5432, - database: 'unleash', + database: process.env.UNLEASH_DATABASE_NAME || 'unleash', + schema: process.env.UNLEASH_DATABASE_SCHEMA, ssl: false, }, server: { @@ -29,9 +31,8 @@ process.nextTick(async () => { enable: false, }, experimental: { - metricsV2: { - enabled: true, - }, + metricsV2: { enabled: true }, + segments: experimentalSegmentsConfig(), }, authentication: { initApiTokens: [ diff --git a/src/test/config/test-config.ts b/src/test/config/test-config.ts index 864872a5432..d81cb7797fc 100644 --- a/src/test/config/test-config.ts +++ b/src/test/config/test-config.ts @@ -7,6 +7,7 @@ import { import getLogger from '../fixtures/no-logger'; import { createConfig } from '../../lib/create-config'; +import { experimentalSegmentsConfig } from '../../lib/experimental'; function mergeAll(objects: Partial[]): T { return merge.all(objects.filter((i) => i)); @@ -20,6 +21,9 @@ export function createTestConfig(config?: IUnleashOptions): IUnleashConfig { session: { db: false, }, + experimental: { + segments: experimentalSegmentsConfig(), + }, versionCheck: { enable: false }, }; const options = mergeAll([testConfig, config]); diff --git a/src/test/e2e/api/admin/constraints.e2e.test.ts b/src/test/e2e/api/admin/constraints.e2e.test.ts new file mode 100644 index 00000000000..11a30f17d5b --- /dev/null +++ b/src/test/e2e/api/admin/constraints.e2e.test.ts @@ -0,0 +1,36 @@ +import dbInit from '../../helpers/database-init'; +import getLogger from '../../../fixtures/no-logger'; +import { setupApp } from '../../helpers/test-helper'; + +let app; +let db; + +const PATH = '/api/admin/constraints/validate'; + +beforeAll(async () => { + db = await dbInit('constraints', getLogger); + app = await setupApp(db.stores); +}); + +afterAll(async () => { + await app.destroy(); + await db.destroy(); +}); + +test('should reject invalid constraints', async () => { + await app.request.post(PATH).send({}).expect(400); + await app.request.post(PATH).send({ a: 1 }).expect(400); + await app.request.post(PATH).send({ operator: 'IN' }).expect(400); + await app.request.post(PATH).send({ contextName: 'a' }).expect(400); +}); + +test('should accept valid constraints', async () => { + await app.request + .post(PATH) + .send({ contextName: 'environment', operator: 'NUM_EQ', value: 1 }) + .expect(204); + await app.request + .post(PATH) + .send({ contextName: 'environment', operator: 'IN', values: ['a'] }) + .expect(204); +}); diff --git a/src/test/e2e/api/admin/state.e2e.test.ts b/src/test/e2e/api/admin/state.e2e.test.ts index e2f4ffc51f7..2b48827ecaa 100644 --- a/src/test/e2e/api/admin/state.e2e.test.ts +++ b/src/test/e2e/api/admin/state.e2e.test.ts @@ -2,6 +2,7 @@ import dbInit, { ITestDb } from '../../helpers/database-init'; import { IUnleashTest, setupApp } from '../../helpers/test-helper'; import getLogger from '../../../fixtures/no-logger'; import { DEFAULT_ENV } from '../../../../lib/util/constants'; +import { collectIds } from '../../../../lib/util/collect-ids'; const importData = require('../../../examples/import.json'); @@ -321,3 +322,18 @@ test(`Importing version 2 replaces :global: environment with 'default'`, async ( expect(feature.environments).toHaveLength(1); expect(feature.environments[0].name).toBe(DEFAULT_ENV); }); + +test(`should import segments and connect them to feature strategies`, async () => { + await app.request + .post('/api/admin/state/import') + .attach('file', 'src/test/examples/exported-segments.json') + .expect(202); + + const allSegments = await app.services.segmentService.getAll(); + const activeSegments = await app.services.segmentService.getActive(); + + expect(allSegments.length).toEqual(2); + expect(collectIds(allSegments)).toEqual([1, 2]); + expect(activeSegments.length).toEqual(1); + expect(collectIds(activeSegments)).toEqual([1]); +}); diff --git a/src/test/e2e/api/client/segment.e2e.test.ts b/src/test/e2e/api/client/segment.e2e.test.ts new file mode 100644 index 00000000000..9f75cdd7555 --- /dev/null +++ b/src/test/e2e/api/client/segment.e2e.test.ts @@ -0,0 +1,147 @@ +import dbInit, { ITestDb } from '../../helpers/database-init'; +import getLogger from '../../../fixtures/no-logger'; +import { IUnleashTest, setupApp } from '../../helpers/test-helper'; +import { collectIds } from '../../../../lib/util/collect-ids'; +import { + IConstraint, + IFeatureToggleClient, + ISegment, +} from '../../../../lib/types/model'; +import { randomId } from '../../../../lib/util/random-id'; +import User from '../../../../lib/types/user'; + +let db: ITestDb; +let app: IUnleashTest; + +const FEATURES_ADMIN_BASE_PATH = '/api/admin/features'; +const FEATURES_CLIENT_BASE_PATH = '/api/client/features'; + +const fetchSegments = (): Promise => { + return app.services.segmentService.getAll(); +}; + +const fetchFeatures = (): Promise => { + return app.request + .get(FEATURES_ADMIN_BASE_PATH) + .expect(200) + .then((res) => res.body.features); +}; + +const fetchClientFeatures = (): Promise => { + return app.request + .get(FEATURES_CLIENT_BASE_PATH) + .expect(200) + .then((res) => res.body.features); +}; + +const fetchClientSegmentsActive = (): Promise => { + return app.request + .get('/api/client/segments/active') + .expect(200) + .then((res) => res.body.segments); +}; + +const createSegment = (postData: object): Promise => { + const user = { email: 'test@example.com' } as User; + return app.services.segmentService.create(postData, user); +}; + +const createFeatureToggle = ( + postData: object, + expectStatusCode = 201, +): Promise => { + return app.request + .post(FEATURES_ADMIN_BASE_PATH) + .send(postData) + .expect(expectStatusCode); +}; + +const addSegmentToStrategy = ( + segmentId: number, + strategyId: string, +): Promise => { + return app.services.segmentService.addToStrategy(segmentId, strategyId); +}; + +const mockFeatureToggle = (): object => { + return { + name: randomId(), + strategies: [{ name: randomId(), constraints: [], parameters: {} }], + }; +}; + +const mockConstraints = (): IConstraint[] => { + return Array.from({ length: 5 }).map(() => ({ + values: ['x', 'y', 'z'], + operator: 'IN', + contextName: 'a', + })); +}; + +beforeAll(async () => { + db = await dbInit('segments', getLogger); + app = await setupApp(db.stores); +}); + +afterAll(async () => { + await app.destroy(); + await db.destroy(); +}); + +afterEach(async () => { + await db.stores.segmentStore.deleteAll(); + await db.stores.featureToggleStore.deleteAll(); +}); + +test('should add segments to features as constraints', async () => { + const constraints = mockConstraints(); + await createSegment({ name: 'S1', constraints }); + await createSegment({ name: 'S2', constraints }); + await createSegment({ name: 'S3', constraints }); + await createFeatureToggle(mockFeatureToggle()); + await createFeatureToggle(mockFeatureToggle()); + await createFeatureToggle(mockFeatureToggle()); + const [feature1, feature2, feature3] = await fetchFeatures(); + const [segment1, segment2, segment3] = await fetchSegments(); + + await addSegmentToStrategy(segment1.id, feature1.strategies[0].id); + await addSegmentToStrategy(segment2.id, feature1.strategies[0].id); + await addSegmentToStrategy(segment2.id, feature2.strategies[0].id); + await addSegmentToStrategy(segment3.id, feature1.strategies[0].id); + await addSegmentToStrategy(segment3.id, feature2.strategies[0].id); + await addSegmentToStrategy(segment3.id, feature3.strategies[0].id); + + const clientFeatures = await fetchClientFeatures(); + const clientStrategies = clientFeatures.flatMap((f) => f.strategies); + const clientConstraints = clientStrategies.flatMap((s) => s.constraints); + const clientValues = clientConstraints.flatMap((c) => c.values); + const uniqueValues = [...new Set(clientValues)]; + + expect(clientFeatures.length).toEqual(3); + expect(clientStrategies.length).toEqual(3); + expect(clientConstraints.length).toEqual(5 * 6); + expect(clientValues.length).toEqual(5 * 6 * 3); + expect(uniqueValues.length).toEqual(3); +}); + +test('should list active segments', async () => { + const constraints = mockConstraints(); + await createSegment({ name: 'S1', constraints }); + await createSegment({ name: 'S2', constraints }); + await createSegment({ name: 'S3', constraints }); + await createFeatureToggle(mockFeatureToggle()); + await createFeatureToggle(mockFeatureToggle()); + await createFeatureToggle(mockFeatureToggle()); + const [feature1, feature2] = await fetchFeatures(); + const [segment1, segment2] = await fetchSegments(); + + await addSegmentToStrategy(segment1.id, feature1.strategies[0].id); + await addSegmentToStrategy(segment2.id, feature1.strategies[0].id); + await addSegmentToStrategy(segment2.id, feature2.strategies[0].id); + + const clientSegments = await fetchClientSegmentsActive(); + + expect(collectIds(clientSegments)).toEqual( + collectIds([segment1, segment2]), + ); +}); diff --git a/src/test/e2e/seed/segment.seed.ts b/src/test/e2e/seed/segment.seed.ts new file mode 100644 index 00000000000..9bdf5e6f90f --- /dev/null +++ b/src/test/e2e/seed/segment.seed.ts @@ -0,0 +1,149 @@ +import dbInit from '../helpers/database-init'; +import getLogger from '../../fixtures/no-logger'; +import assert from 'assert'; +import User from '../../../lib/types/user'; +import { randomId } from '../../../lib/util/random-id'; +import { + IConstraint, + IFeatureToggleClient, + ISegment, +} from '../../../lib/types/model'; +import { IUnleashTest, setupApp } from '../helpers/test-helper'; + +interface ISeedSegmentSpec { + featuresCount: number; + segmentsPerFeature: number; + constraintsPerSegment: number; + valuesPerConstraint: number; +} + +// The number of items to insert. +const seedSegmentSpec: ISeedSegmentSpec = { + featuresCount: 100, + segmentsPerFeature: 5, + constraintsPerSegment: 1, + valuesPerConstraint: 100, +}; + +// The database schema to populate. +const seedSchema = 'seed'; + +const fetchSegments = (app: IUnleashTest): Promise => { + return app.services.segmentService.getAll(); +}; + +const fetchFeatures = (app: IUnleashTest): Promise => { + return app.request + .get('/api/admin/features') + .expect(200) + .then((res) => res.body.features); +}; + +const createSegment = ( + app: IUnleashTest, + postData: object, +): Promise => { + const user = { email: 'test@example.com' } as User; + return app.services.segmentService.create(postData, user); +}; + +const createFeatureToggle = ( + app: IUnleashTest, + postData: object, + expectStatusCode = 201, +): Promise => { + return app.request + .post('/api/admin/features') + .send(postData) + .expect(expectStatusCode); +}; + +const addSegmentToStrategy = ( + app: IUnleashTest, + segmentId: number, + strategyId: string, +): Promise => { + return app.services.segmentService.addToStrategy(segmentId, strategyId); +}; + +const mockFeatureToggle = ( + overrides?: Partial, +): Partial => { + return { + name: randomId(), + strategies: [{ name: randomId(), constraints: [], parameters: {} }], + ...overrides, + }; +}; + +const seedConstraints = (spec: ISeedSegmentSpec): IConstraint[] => { + return Array.from({ length: spec.constraintsPerSegment }).map(() => ({ + values: Array.from({ length: spec.valuesPerConstraint }).map(() => + randomId().substring(0, 16), + ), + operator: 'IN', + contextName: 'x', + })); +}; + +const seedSegments = (spec: ISeedSegmentSpec): Partial[] => { + return Array.from({ length: spec.segmentsPerFeature }).map((v, i) => { + return { + name: `${seedSchema}_segment_${i}`, + constraints: seedConstraints(spec), + }; + }); +}; + +const seedFeatures = ( + spec: ISeedSegmentSpec, +): Partial[] => { + return Array.from({ length: spec.featuresCount }).map((v, i) => { + return mockFeatureToggle({ + name: `${seedSchema}_feature_${i}`, + }); + }); +}; + +const seedSegmentsDatabase = async ( + app: IUnleashTest, + spec: ISeedSegmentSpec, +): Promise => { + await Promise.all( + seedSegments(spec).map((seed) => { + return createSegment(app, seed); + }), + ); + + await Promise.all( + seedFeatures(spec).map((seed) => { + return createFeatureToggle(app, seed); + }), + ); + + const features = await fetchFeatures(app); + const segments = await fetchSegments(app); + assert(features.length === spec.featuresCount); + assert(segments.length === spec.segmentsPerFeature); + + const addSegment = (feature: IFeatureToggleClient, segment: ISegment) => { + return addSegmentToStrategy(app, segment.id, feature.strategies[0].id); + }; + + for (const feature of features) { + await Promise.all( + segments.map((segment) => addSegment(feature, segment)), + ); + } +}; + +const main = async (): Promise => { + const db = await dbInit(seedSchema, getLogger); + const app = await setupApp(db.stores); + + await seedSegmentsDatabase(app, seedSegmentSpec); + await app.destroy(); + await db.destroy(); +}; + +main().catch(console.error); diff --git a/src/test/examples/exported-segments.json b/src/test/examples/exported-segments.json new file mode 100644 index 00000000000..c2b0814133d --- /dev/null +++ b/src/test/examples/exported-segments.json @@ -0,0 +1,77 @@ +{ + "version": 3, + "features": [ + { + "name": "seed_feature_1", + "description": null, + "type": "release", + "project": "default", + "stale": false, + "variants": [], + "createdAt": "2022-03-17T20:18:55.643Z", + "lastSeenAt": null, + "impressionData": false + } + ], + "featureStrategies": [ + { + "id": "854adc48-7d4e-4433-9c36-5a4cd15491dc", + "featureName": "seed_feature_1", + "projectId": "default", + "environment": "default", + "strategyName": "bc555d20-cfef-490f-a30d-d922e5675b0e", + "parameters": {}, + "constraints": [], + "createdAt": "2022-03-17T20:18:55.667Z" + } + ], + "featureEnvironments": [ + { + "enabled": true, + "featureName": "seed_feature_1", + "environment": "default" + } + ], + "segments": [ + { + "id": 1, + "name": "seed_segment_1", + "description": null, + "constraints": [ + { + "values": [ + "878a9f99-96a0-46", + "894fc669-f53d-43" + ], + "operator": "IN", + "contextName": "x" + } + ], + "createdBy": "some-user", + "createdAt": "2022-03-17T20:18:55.696Z" + }, + { + "id": 2, + "name": "seed_segment_2", + "description": null, + "constraints": [ + { + "values": [ + "894fc669-f53d-43", + "15fecdbf-d3b6-48" + ], + "operator": "IN", + "contextName": "x" + } + ], + "createdBy": "some-user", + "createdAt": "2022-03-17T20:18:55.696Z" + } + ], + "featureStrategySegments": [ + { + "featureStrategyId": "854adc48-7d4e-4433-9c36-5a4cd15491dc", + "segmentId": 1 + } + ] +} diff --git a/src/test/fixtures/fake-feature-strategies-store.ts b/src/test/fixtures/fake-feature-strategies-store.ts index 36941061c37..a8d8060be29 100644 --- a/src/test/fixtures/fake-feature-strategies-store.ts +++ b/src/test/fixtures/fake-feature-strategies-store.ts @@ -270,6 +270,10 @@ export default class FakeFeatureStrategiesStore ): Promise { return Promise.resolve(enabled); } + + getStrategiesBySegment(): Promise { + throw new Error('Method not implemented.'); + } } module.exports = FakeFeatureStrategiesStore; diff --git a/src/test/fixtures/fake-segment-store.ts b/src/test/fixtures/fake-segment-store.ts new file mode 100644 index 00000000000..644210282a1 --- /dev/null +++ b/src/test/fixtures/fake-segment-store.ts @@ -0,0 +1,54 @@ +import { ISegmentStore } from '../../lib/types/stores/segment-store'; +import { IFeatureStrategySegment, ISegment } from '../../lib/types/model'; + +export default class FakeSegmentStore implements ISegmentStore { + create(): Promise { + throw new Error('Method not implemented.'); + } + + async delete(): Promise { + return; + } + + async deleteAll(): Promise { + return; + } + + async exists(): Promise { + return false; + } + + get(): Promise { + throw new Error('Method not implemented.'); + } + + async getAll(): Promise { + return []; + } + + async getActive(): Promise { + return []; + } + + async getByStrategy(): Promise { + return []; + } + + update(): Promise { + throw new Error('Method not implemented.'); + } + + addToStrategy(): Promise { + throw new Error('Method not implemented.'); + } + + removeFromStrategy(): Promise { + throw new Error('Method not implemented.'); + } + + async getAllFeatureStrategySegments(): Promise { + return []; + } + + destroy(): void {} +} diff --git a/src/test/fixtures/store.ts b/src/test/fixtures/store.ts index d2724626ba1..9b378c2368f 100644 --- a/src/test/fixtures/store.ts +++ b/src/test/fixtures/store.ts @@ -25,6 +25,7 @@ import FakeFeatureToggleClientStore from './fake-feature-toggle-client-store'; import FakeClientMetricsStoreV2 from './fake-client-metrics-store-v2'; import FakeUserSplashStore from './fake-user-splash-store'; import FakeRoleStore from './fake-role-store'; +import FakeSegmentStore from './fake-segment-store'; const createStores: () => IUnleashStores = () => { const db = { @@ -61,6 +62,7 @@ const createStores: () => IUnleashStores = () => { sessionStore: new FakeSessionStore(), userSplashStore: new FakeUserSplashStore(), roleStore: new FakeRoleStore(), + segmentStore: new FakeSegmentStore(), }; };